From 666dfdcd77e96b2c8e644156a35643ec2be59ee6 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Tue, 15 Sep 2026 23:04:14 +0800 Subject: [PATCH 01/11] feat(physical-optimizer): let a rule opt into skipping an unchanged re-run The logical optimizer iterates and stops on convergence via LogicalPlanSignature. The physical optimizer runs its list once, which suits the default chain -- everything able to invalidate distribution or ordering requirements is deliberately ordered before the single EnsureRequirements. Custom rule lists do not have that luxury: a rewrite inserted after that point invalidates requirements again and needs its own enforcement pass, and some of those passes run on a plan no preceding rule touched. Rules already return their input Arc untouched when they have nothing to do, so pointer identity is an exact, allocation-free 'nothing happened' signal. A rule can now declare skip_if_unchanged(); when the config flag datafusion.optimizer.skip_unchanged_physical_rules is on, the optimizer remembers the plan each opted-in rule returned and skips the call when handed back that same object. The memo lives in the optimization run, keyed by rule name, so nothing leaks across queries (rule instances are shared) and a rule listed twice as two instances still matches. Debug builds run a skipped rule anyway and assert it changed nothing, so a rule that declares purity without having it fails a test rather than a query -- Spark is the only surveyed engine that checks this, and the engines that rely on counters instead have public incidents from non-idempotent rules. Both flags default to off, so nothing changes until a rule and the session agree. --- datafusion/common/src/config.rs | 12 ++ datafusion/core/src/physical_planner.rs | 157 ++++++++++++++++++- datafusion/session/src/physical_optimizer.rs | 25 +++ docs/source/user-guide/configs.md | 1 + 4 files changed, 194 insertions(+), 1 deletion(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 360586b0e9bae..bc6db18c9279b 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1597,6 +1597,18 @@ config_namespace! { /// repartitioning to increase parallelism to leverage more CPU cores pub enable_round_robin_repartition: bool, default = true + /// When set to true, a physical optimizer rule that opts in via + /// [`PhysicalOptimizerRule::skip_if_unchanged`] is skipped when its + /// input is the very plan it returned last time, since re-running a + /// pure rule on its own output cannot change anything. + /// + /// The default rule list has no repeated rules, so this matters for + /// custom rule lists (`with_physical_optimizer_rules`) that enforce + /// requirements again after their own rewrites. + /// + /// [`PhysicalOptimizerRule::skip_if_unchanged`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged + pub skip_unchanged_physical_rules: bool, default = false + /// When set to true, the optimizer will attempt to perform limit operations /// during aggregations, if possible pub enable_topk_aggregation: bool, default = true diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 77997c619e5ce..ec672a2c0a2f0 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2965,13 +2965,62 @@ impl DefaultPhysicalPlanner { let optimizer_context = SessionOptimizerContext { session: session_state, }; + // Remembers the plan each opted-in rule last returned, so the rule can + // be skipped when handed back that exact object. Keyed by rule name + // rather than position, because a repeated rule is normally a second + // instance rather than the same one, and it is the rule's identity + // that makes re-running it pointless. Scoped to this call: rule + // instances are shared between queries, so this must not live on the + // rule itself. + let skip_unchanged = session_state + .config_options() + .optimizer + .skip_unchanged_physical_rules; + let mut last_outputs: HashMap<&str, Arc> = HashMap::new(); + for optimizer in optimizers { + if skip_unchanged && optimizer.skip_if_unchanged() { + if let Some(last) = last_outputs.get(optimizer.name()) + && Arc::ptr_eq(last, &new_plan) + { + // The rule produced this exact plan and nothing since has + // replaced it, so running it again cannot change anything. + // Debug builds verify that claim rather than trusting it. + #[cfg(debug_assertions)] + { + let rerun = optimizer + .optimize_with_context( + Arc::clone(&new_plan), + &optimizer_context, + ) + .map_err(|e| { + DataFusionError::Context( + optimizer.name().to_string(), + Box::new(e), + ) + })?; + debug_assert!( + Arc::ptr_eq(&rerun, &new_plan), + "PhysicalOptimizer rule '{}' declares skip_if_unchanged() \ + but rewrote a plan it had already produced; the rule is \ + not a pure function of its input", + optimizer.name(), + ); + } + observer(new_plan.as_ref(), optimizer.as_ref()); + continue; + } + } + let before_schema = new_plan.schema(); new_plan = optimizer .optimize_with_context(new_plan, &optimizer_context) .map_err(|e| { DataFusionError::Context(optimizer.name().to_string(), Box::new(e)) })?; + if skip_unchanged && optimizer.skip_if_unchanged() { + last_outputs.insert(optimizer.name(), Arc::clone(&new_plan)); + } // This only checks the schema in release build, and performs additional checks in debug mode. OptimizationInvariantChecker::new(optimizer) @@ -3368,7 +3417,7 @@ mod tests { use std::fmt::{self, Debug}; use std::mem::size_of_val; use std::ops::{BitAnd, Not}; - use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; use super::*; use crate::datasource::MemTable; @@ -3694,6 +3743,112 @@ mod tests { Ok(()) } + /// Counts its invocations and hands the plan back untouched, mimicking a + /// rule that finds nothing to do on an already-satisfied plan. + #[derive(Debug)] + struct CountingNoopRule { + calls: Arc, + skip_if_unchanged: bool, + } + + impl PhysicalOptimizerRule for CountingNoopRule { + fn optimize( + &self, + plan: Arc, + _config: &ConfigOptions, + ) -> Result> { + self.calls.fetch_add(1, AtomicOrdering::Relaxed); + Ok(plan) + } + + fn name(&self) -> &str { + "counting_noop_rule" + } + + fn skip_if_unchanged(&self) -> bool { + self.skip_if_unchanged + } + + fn schema_check(&self) -> bool { + true + } + } + + /// Expected invocations of a rule listed twice. Debug builds verify the + /// idempotence claim by running a skipped rule anyway and asserting it + /// changed nothing, so the call still happens there — what the skip saves + /// in debug is nothing, and in release it is the whole second pass. + const SKIPPED_CALLS: usize = if cfg!(debug_assertions) { 2 } else { 1 }; + + async fn run_repeated_rule( + skip_if_unchanged: bool, + skip_enabled: bool, + ) -> Result { + let calls = Arc::new(AtomicUsize::new(0)); + let rule = || { + Arc::new(CountingNoopRule { + calls: Arc::clone(&calls), + skip_if_unchanged, + }) as Arc + }; + let mut config = SessionConfig::new(); + config.options_mut().optimizer.skip_unchanged_physical_rules = skip_enabled; + // The same rule twice, which is the shape a custom rule list takes + // when a rewrite between the two passes may or may not fire. + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_physical_optimizer_rules(vec![rule(), rule()]) + .build(); + let ctx = SessionContext::new_with_state(state); + let logical_plan = LogicalPlanBuilder::empty(false).build()?; + ctx.state().create_physical_plan(&logical_plan).await?; + Ok(calls.load(AtomicOrdering::Relaxed)) + } + + /// A rule that opted in is called once instead of twice: the second entry + /// receives the exact plan the first returned. + #[tokio::test] + async fn skip_unchanged_skips_the_repeated_pass() -> Result<()> { + assert_eq!(run_repeated_rule(true, true).await?, SKIPPED_CALLS); + Ok(()) + } + + /// Off by default, and opting in without the config flag changes nothing: + /// both entries run, exactly as before this feature existed. + #[tokio::test] + async fn skip_unchanged_is_inert_unless_both_sides_agree() -> Result<()> { + assert_eq!(run_repeated_rule(false, true).await?, 2); + assert_eq!(run_repeated_rule(true, false).await?, 2); + assert_eq!(run_repeated_rule(false, false).await?, 2); + Ok(()) + } + + /// The memo is per optimization run, not per rule instance: planning a + /// second query must not let the first query's plan suppress a call. + #[tokio::test] + async fn skip_unchanged_does_not_leak_between_plans() -> Result<()> { + let calls = Arc::new(AtomicUsize::new(0)); + let mut config = SessionConfig::new(); + config.options_mut().optimizer.skip_unchanged_physical_rules = true; + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_physical_optimizer_rules(vec![Arc::new(CountingNoopRule { + calls: Arc::clone(&calls), + skip_if_unchanged: true, + })]) + .build(); + let ctx = SessionContext::new_with_state(state); + let logical_plan = LogicalPlanBuilder::empty(false).build()?; + ctx.state().create_physical_plan(&logical_plan).await?; + ctx.state().create_physical_plan(&logical_plan).await?; + // Once per plan; a rule-level memo would have suppressed the second. + // The rule is listed once here, so the debug self-check never fires. + assert_eq!(calls.load(AtomicOrdering::Relaxed), 2); + Ok(()) + } + async fn aggregate_explain(logical_plan: &LogicalPlan) -> Result { let physical_plan = plan(logical_plan).await?; Ok(displayable(physical_plan.as_ref()).indent(true).to_string()) diff --git a/datafusion/session/src/physical_optimizer.rs b/datafusion/session/src/physical_optimizer.rs index 751a8e12d93ed..7a7db071be229 100644 --- a/datafusion/session/src/physical_optimizer.rs +++ b/datafusion/session/src/physical_optimizer.rs @@ -76,6 +76,31 @@ pub trait PhysicalOptimizerRule: Debug + std::any::Any { /// A human readable name for this optimizer rule fn name(&self) -> &str; + /// Whether this rule is a pure function of the plan it is given, so that + /// running it again on a plan object it previously returned cannot change + /// anything. + /// + /// When a rule opts in, the optimizer remembers the plan the rule last + /// returned and skips the call when handed back that exact object. Plans + /// are reference counted and a rule returns its input untouched when it + /// has nothing to do, so this is an exact, allocation-free signal — no + /// hashing or structural comparison is involved. + /// + /// This is off by default and only consulted when + /// `datafusion.optimizer.skip_unchanged_physical_rules` is enabled. It + /// pays off for rule lists that run the same rule more than once, which + /// is common when downstream rewrites are inserted after the built-in + /// requirement enforcement and each needs its requirements re-enforced. + /// + /// Leave this `false` for any rule whose output depends on state outside + /// the plan — session state that can change between invocations, + /// counters, randomness — since the same input would no longer imply the + /// same output. Debug builds verify the claim: when a skip would fire, + /// the rule is run anyway and the result is asserted to be unchanged. + fn skip_if_unchanged(&self) -> bool { + false + } + /// A flag to indicate whether the physical planner should validate that the rule will not /// change the schema of the plan after the rewriting. /// Some of the optimization rules might change the nullable properties of the schema diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 0085d4ac7c1fa..9a46837e9ec23 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -147,6 +147,7 @@ The following configuration settings are available: | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | | datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | | datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | +| datafusion.optimizer.skip_unchanged_physical_rules | false | When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`] is skipped when its input is the very plan it returned last time, since re-running a pure rule on its own output cannot change anything. The default rule list has no repeated rules, so this matters for custom rule lists (`with_physical_optimizer_rules`) that enforce requirements again after their own rewrites. [`PhysicalOptimizerRule::skip_if_unchanged`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged | | datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | | datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | | datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | From 2c6ebf24feb59bcc0314795080e16fa42a09bd16 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 10:53:22 +0800 Subject: [PATCH 02/11] test: cover an interleaved enforce/rewrite chain The three existing tests exercise the mechanism; this one exercises the shape it exists for -- enforcement passes separated by rewrites, where only the passes following a rewrite that actually fired have work to do. --- datafusion/core/src/physical_planner.rs | 75 +++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index ec672a2c0a2f0..478f4dd0fb113 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -3824,6 +3824,81 @@ mod tests { Ok(()) } + /// Replaces the plan with an equivalent new object, standing in for a + /// rewrite that fires only for some queries. + #[derive(Debug)] + struct RewritingRule { + name: &'static str, + rewrite: bool, + } + + impl PhysicalOptimizerRule for RewritingRule { + fn optimize( + &self, + plan: Arc, + _config: &ConfigOptions, + ) -> Result> { + if self.rewrite { + Ok(Arc::new(EmptyExec::new(plan.schema()))) + } else { + Ok(plan) + } + } + + fn name(&self) -> &str { + self.name + } + + fn schema_check(&self) -> bool { + true + } + } + + /// The shape this feature exists for: a chain that enforces requirements, + /// applies its own rewrites, and enforces again after each one. Only the + /// enforcement passes that follow a rewrite which actually fired have work + /// to do; the others receive the plan the previous enforcement produced. + /// + /// Here the first rewrite fires and the second does not, so of three + /// enforcement passes exactly two must run. + #[tokio::test] + async fn skip_unchanged_handles_an_interleaved_chain() -> Result<()> { + let calls = Arc::new(AtomicUsize::new(0)); + let enforce = || { + Arc::new(CountingNoopRule { + calls: Arc::clone(&calls), + skip_if_unchanged: true, + }) as Arc + }; + let rewrite = |name, rewrite| { + Arc::new(RewritingRule { name, rewrite }) + as Arc + }; + + let mut config = SessionConfig::new(); + config.options_mut().optimizer.skip_unchanged_physical_rules = true; + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_physical_optimizer_rules(vec![ + enforce(), // runs: nothing memoized yet + rewrite("rewrite_that_fires", true), + enforce(), // runs: the plan changed + rewrite("rewrite_that_does_not", false), + enforce(), // skipped: plan is unchanged + ]) + .build(); + let ctx = SessionContext::new_with_state(state); + let logical_plan = LogicalPlanBuilder::empty(false).build()?; + ctx.state().create_physical_plan(&logical_plan).await?; + + // Two enforcement passes have real work; the third is skipped (and in + // debug builds re-run by the self-check, which is why this counts + // against SKIPPED_CALLS rather than a literal). + assert_eq!(calls.load(AtomicOrdering::Relaxed), 2 + (SKIPPED_CALLS - 1)); + Ok(()) + } + /// The memo is per optimization run, not per rule instance: planning a /// second query must not let the first query's plan suppress a call. #[tokio::test] From cdb74f141b5465173f2d9f8e38f45e695030da93 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 10:56:34 +0800 Subject: [PATCH 03/11] feat: opt EnsureRequirements into skip_if_unchanged The rule derives everything from the plan and the config, and the config is fixed for an optimization run, so running it again on a plan it just produced cannot change anything. Without this the new trait method has no implementor in tree and rule lists that enforce requirements after their own rewrites -- the case the feature exists for -- get nothing. --- .../physical-optimizer/src/ensure_requirements/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs index 2bc57915b2315..493b912be7e19 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs @@ -260,6 +260,15 @@ impl PhysicalOptimizerRule for EnsureRequirements { "EnsureRequirements" } + /// This rule derives everything it does from the plan and the config, and + /// the config is fixed for an optimization run, so re-running it on a plan + /// it just produced cannot change anything. Rule lists that enforce + /// requirements again after their own rewrites can therefore skip the + /// passes whose input nothing has touched. + fn skip_if_unchanged(&self) -> bool { + true + } + fn schema_check(&self) -> bool { true } From 6b1731c8bfdfb976662ca074198745c4fbf1a573 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 11:14:07 +0800 Subject: [PATCH 04/11] refactor: build the skip memo only when the feature is enabled The map was allocated on every physical planning run even with the feature off. Making it an Option keeps the disabled path free of it. --- datafusion/core/src/physical_planner.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 478f4dd0fb113..3125bb8f644bb 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2972,15 +2972,17 @@ impl DefaultPhysicalPlanner { // that makes re-running it pointless. Scoped to this call: rule // instances are shared between queries, so this must not live on the // rule itself. - let skip_unchanged = session_state + let mut last_outputs = session_state .config_options() .optimizer - .skip_unchanged_physical_rules; - let mut last_outputs: HashMap<&str, Arc> = HashMap::new(); + .skip_unchanged_physical_rules + .then(HashMap::<&str, Arc>::new); for optimizer in optimizers { - if skip_unchanged && optimizer.skip_if_unchanged() { - if let Some(last) = last_outputs.get(optimizer.name()) + if last_outputs.is_some() && optimizer.skip_if_unchanged() { + if let Some(last) = last_outputs + .as_ref() + .and_then(|memo| memo.get(optimizer.name())) && Arc::ptr_eq(last, &new_plan) { // The rule produced this exact plan and nothing since has @@ -3018,8 +3020,10 @@ impl DefaultPhysicalPlanner { .map_err(|e| { DataFusionError::Context(optimizer.name().to_string(), Box::new(e)) })?; - if skip_unchanged && optimizer.skip_if_unchanged() { - last_outputs.insert(optimizer.name(), Arc::clone(&new_plan)); + if let Some(memo) = last_outputs.as_mut() + && optimizer.skip_if_unchanged() + { + memo.insert(optimizer.name(), Arc::clone(&new_plan)); } // This only checks the schema in release build, and performs additional checks in debug mode. From 914a3915302557dd4fe629f73fd141150d3e8be9 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 11:22:41 +0800 Subject: [PATCH 05/11] docs: state that wrapper rules must forward skip_if_unchanged The optimizer consults the rule it holds, so a rule that runs other rules inside its own optimize() decides for all of them. A wrapper leaving this at the default silently opts its inner rules out of the skip, which is the same trap #25316 describes for schema_check(). Say so where an implementor reads it, with the all() form, since a wrapper is skippable only if every rule it would have run is. --- datafusion/session/src/physical_optimizer.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datafusion/session/src/physical_optimizer.rs b/datafusion/session/src/physical_optimizer.rs index 7a7db071be229..6d35e1fd870c7 100644 --- a/datafusion/session/src/physical_optimizer.rs +++ b/datafusion/session/src/physical_optimizer.rs @@ -97,6 +97,20 @@ pub trait PhysicalOptimizerRule: Debug + std::any::Any { /// counters, randomness — since the same input would no longer imply the /// same output. Debug builds verify the claim: when a skip would fire, /// the rule is run anyway and the result is asserted to be unchanged. + /// + /// The optimizer reads this from the rule it holds, so a rule that runs + /// *other* rules inside its own [`optimize`] must forward their answer — + /// in practice `all()` over the rules it wraps, since the wrapper is only + /// skippable if every rule it would have run is. A wrapper that leaves + /// this at the default silently opts its inner rules out: + /// + /// ```text + /// fn skip_if_unchanged(&self) -> bool { + /// self.wrapped.iter().all(|rule| rule.skip_if_unchanged()) + /// } + /// ``` + /// + /// [`optimize`]: PhysicalOptimizerRule::optimize fn skip_if_unchanged(&self) -> bool { false } From 8ac5ecccade673ea5dd91a6cdf6500c344fb1ce7 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 11:54:45 +0800 Subject: [PATCH 06/11] fix: register skip_unchanged_physical_rules with SHOW ALL The option was added to OptimizerOptions without the matching rows in information_schema.slt, which SHOW ALL asserts exhaustively. --- datafusion/sqllogictest/test_files/information_schema.slt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index b270eba99d7b0..9dff7abc4d049 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -339,6 +339,7 @@ datafusion.optimizer.repartition_joins true datafusion.optimizer.repartition_sorts true datafusion.optimizer.repartition_windows true datafusion.optimizer.skip_failed_rules false +datafusion.optimizer.skip_unchanged_physical_rules false datafusion.optimizer.subset_repartition_threshold 4 datafusion.optimizer.top_down_join_key_reordering true datafusion.optimizer.use_statistics_registry false @@ -500,6 +501,7 @@ datafusion.optimizer.repartition_joins true Should DataFusion repartition data u datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail +datafusion.optimizer.skip_unchanged_physical_rules false When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) is skipped when the plan handed to it is the very plan it returned last time, since an idempotent rule run on its own output only arrives at that same plan again. "Nothing changed" is tested by pointer identity, which is exact and free: plans are reference counted, and a rule that finds nothing to do returns its input untouched, so an undisturbed stretch of the rule list carries the same object through to the next pass. Physical rules run as a fixed sequence with no fixpoint loop, so this matters for rule lists that hold the same rule more than once. The built-in list holds none twice, which is why this is off by default; it is aimed at lists installed through `SessionStateBuilder::with_physical_optimizer_rules`, where custom rewrites are inserted after the built-in requirement enforcement and each of them needs requirements enforced again. datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys datafusion.optimizer.use_statistics_registry false (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). From dbac5204f40015cc8e0b976602cd3663b6dd1e00 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 11:54:56 +0800 Subject: [PATCH 07/11] fix: check idempotence by plan, not by object identity The debug self-check asserted that re-running a skipped rule returned the same Arc. EnsureRequirements fails that: it rebuilds the tree and hands back a fresh object describing an identical plan, so enabling the optimization over the real rule panicked in debug builds. Identity was the wrong thing to assert. What the skip relies on is that a second pass would arrive at the same plan, which is idempotence; the check now compares the rendered plans. That a rule rebuilds the tree also sharpens the motivation, since the pass being skipped reconstructs the whole plan to end up where it started. Docs in the trait, the config option and EnsureRequirements said 'pure function' and 'returns its input untouched', both of which read as identity, and are corrected to say idempotent. Also collapses the nested if the skip introduced, which clippy rejects. Covers the mechanism with three further tests: a wrapper rule that forwards the opt-in is skipped while one that drops it is not; a query planned through the built-in rules with two extra EnsureRequirements passes appended produces an identical plan with the optimization on and off; and EXPLAIN VERBOSE renders the same output either way, since a skipped rule still reports to the observer. --- datafusion/common/src/config.rs | 23 +- datafusion/core/src/physical_planner.rs | 241 +++++++++++++----- .../src/ensure_requirements/mod.rs | 12 +- datafusion/session/src/physical_optimizer.rs | 26 +- docs/source/user-guide/configs.md | 2 +- 5 files changed, 224 insertions(+), 80 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index bc6db18c9279b..969948bd2a574 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1598,15 +1598,22 @@ config_namespace! { pub enable_round_robin_repartition: bool, default = true /// When set to true, a physical optimizer rule that opts in via - /// [`PhysicalOptimizerRule::skip_if_unchanged`] is skipped when its - /// input is the very plan it returned last time, since re-running a - /// pure rule on its own output cannot change anything. + /// [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) + /// is skipped when the plan handed to it is the very plan it returned + /// last time, since an idempotent rule run on its own output only + /// arrives at that same plan again. "Nothing changed" is tested by + /// pointer identity, which is exact and free: plans are reference + /// counted, and a rule that finds nothing to do returns its input + /// untouched, so an undisturbed stretch of the rule list carries the + /// same object through to the next pass. /// - /// The default rule list has no repeated rules, so this matters for - /// custom rule lists (`with_physical_optimizer_rules`) that enforce - /// requirements again after their own rewrites. - /// - /// [`PhysicalOptimizerRule::skip_if_unchanged`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged + /// Physical rules run as a fixed sequence with no fixpoint loop, so + /// this matters for rule lists that hold the same rule more than once. + /// The built-in list holds none twice, which is why this is off by + /// default; it is aimed at lists installed through + /// `SessionStateBuilder::with_physical_optimizer_rules`, where custom + /// rewrites are inserted after the built-in requirement enforcement + /// and each of them needs requirements enforced again. pub skip_unchanged_physical_rules: bool, default = false /// When set to true, the optimizer will attempt to perform limit operations diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 3125bb8f644bb..f081d63c10940 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2979,39 +2979,39 @@ impl DefaultPhysicalPlanner { .then(HashMap::<&str, Arc>::new); for optimizer in optimizers { - if last_outputs.is_some() && optimizer.skip_if_unchanged() { - if let Some(last) = last_outputs - .as_ref() - .and_then(|memo| memo.get(optimizer.name())) - && Arc::ptr_eq(last, &new_plan) + if let Some(memo) = last_outputs.as_ref() + && optimizer.skip_if_unchanged() + && let Some(last) = memo.get(optimizer.name()) + && Arc::ptr_eq(last, &new_plan) + { + // The rule produced this exact plan and nothing since has + // replaced it, so running it again cannot change anything. + // Debug builds verify that claim rather than trusting it. + #[cfg(debug_assertions)] { - // The rule produced this exact plan and nothing since has - // replaced it, so running it again cannot change anything. - // Debug builds verify that claim rather than trusting it. - #[cfg(debug_assertions)] - { - let rerun = optimizer - .optimize_with_context( - Arc::clone(&new_plan), - &optimizer_context, + let rerun = optimizer + .optimize_with_context(Arc::clone(&new_plan), &optimizer_context) + .map_err(|e| { + DataFusionError::Context( + optimizer.name().to_string(), + Box::new(e), ) - .map_err(|e| { - DataFusionError::Context( - optimizer.name().to_string(), - Box::new(e), - ) - })?; - debug_assert!( - Arc::ptr_eq(&rerun, &new_plan), - "PhysicalOptimizer rule '{}' declares skip_if_unchanged() \ - but rewrote a plan it had already produced; the rule is \ - not a pure function of its input", - optimizer.name(), - ); - } - observer(new_plan.as_ref(), optimizer.as_ref()); - continue; + })?; + // Rules routinely rebuild the tree even where they + // change nothing, so a re-run legitimately hands back a + // fresh object. What has to hold is that it describes the + // same plan. + debug_assert_eq!( + displayable(rerun.as_ref()).indent(true).to_string(), + displayable(new_plan.as_ref()).indent(true).to_string(), + "PhysicalOptimizer rule '{}' declares skip_if_unchanged() \ + but running it on a plan it had already produced changed \ + that plan, so the rule is not idempotent", + optimizer.name(), + ); } + observer(new_plan.as_ref(), optimizer.as_ref()); + continue; } let before_schema = new_plan.schema(); @@ -3461,6 +3461,8 @@ mod tests { use datafusion_functions_aggregate::count::{count_all, count_udaf}; use datafusion_functions_aggregate::expr_fn::sum; use datafusion_physical_expr::EquivalenceProperties; + use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements; + use datafusion_physical_optimizer::optimizer::PhysicalOptimizer; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions}; use datafusion_session::QueryPlanner; @@ -3778,12 +3780,71 @@ mod tests { } } + /// Stands in for the instrumentation wrappers downstream projects put + /// around every rule to time or trace it. The optimizer only ever asks the + /// outermost rule, so a wrapper has to pass the answer through; one that + /// does not silently opts the rule it wraps back out. + #[derive(Debug)] + struct WrappingRule { + inner: Arc, + forwards_skip_if_unchanged: bool, + } + + impl PhysicalOptimizerRule for WrappingRule { + fn optimize( + &self, + plan: Arc, + config: &ConfigOptions, + ) -> Result> { + self.inner.optimize(plan, config) + } + + fn optimize_with_context( + &self, + plan: Arc, + context: &dyn PhysicalOptimizerContext, + ) -> Result> { + self.inner.optimize_with_context(plan, context) + } + + fn name(&self) -> &str { + self.inner.name() + } + + fn skip_if_unchanged(&self) -> bool { + self.forwards_skip_if_unchanged && self.inner.skip_if_unchanged() + } + + fn schema_check(&self) -> bool { + self.inner.schema_check() + } + } + /// Expected invocations of a rule listed twice. Debug builds verify the /// idempotence claim by running a skipped rule anyway and asserting it /// changed nothing, so the call still happens there — what the skip saves /// in debug is nothing, and in release it is the whole second pass. const SKIPPED_CALLS: usize = if cfg!(debug_assertions) { 2 } else { 1 }; + /// A context whose physical rule list is exactly `rules`, with the skip + /// optimization on or off. + fn session_with_rules( + skip_enabled: bool, + rules: Vec>, + ) -> SessionContext { + let mut config = SessionConfig::new(); + config.options_mut().optimizer.skip_unchanged_physical_rules = skip_enabled; + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_physical_optimizer_rules(rules) + .build(); + SessionContext::new_with_state(state) + } + + /// Plans the same rule twice, which is the shape a custom rule list takes + /// when a rewrite between the two passes may or may not fire, and reports + /// how many times the rule was actually asked to optimize. async fn run_repeated_rule( skip_if_unchanged: bool, skip_enabled: bool, @@ -3795,16 +3856,7 @@ mod tests { skip_if_unchanged, }) as Arc }; - let mut config = SessionConfig::new(); - config.options_mut().optimizer.skip_unchanged_physical_rules = skip_enabled; - // The same rule twice, which is the shape a custom rule list takes - // when a rewrite between the two passes may or may not fire. - let state = SessionStateBuilder::new() - .with_config(config) - .with_default_features() - .with_physical_optimizer_rules(vec![rule(), rule()]) - .build(); - let ctx = SessionContext::new_with_state(state); + let ctx = session_with_rules(skip_enabled, vec![rule(), rule()]); let logical_plan = LogicalPlanBuilder::empty(false).build()?; ctx.state().create_physical_plan(&logical_plan).await?; Ok(calls.load(AtomicOrdering::Relaxed)) @@ -3879,20 +3931,16 @@ mod tests { as Arc }; - let mut config = SessionConfig::new(); - config.options_mut().optimizer.skip_unchanged_physical_rules = true; - let state = SessionStateBuilder::new() - .with_config(config) - .with_default_features() - .with_physical_optimizer_rules(vec![ + let ctx = session_with_rules( + true, + vec![ enforce(), // runs: nothing memoized yet rewrite("rewrite_that_fires", true), enforce(), // runs: the plan changed rewrite("rewrite_that_does_not", false), enforce(), // skipped: plan is unchanged - ]) - .build(); - let ctx = SessionContext::new_with_state(state); + ], + ); let logical_plan = LogicalPlanBuilder::empty(false).build()?; ctx.state().create_physical_plan(&logical_plan).await?; @@ -3908,17 +3956,13 @@ mod tests { #[tokio::test] async fn skip_unchanged_does_not_leak_between_plans() -> Result<()> { let calls = Arc::new(AtomicUsize::new(0)); - let mut config = SessionConfig::new(); - config.options_mut().optimizer.skip_unchanged_physical_rules = true; - let state = SessionStateBuilder::new() - .with_config(config) - .with_default_features() - .with_physical_optimizer_rules(vec![Arc::new(CountingNoopRule { + let ctx = session_with_rules( + true, + vec![Arc::new(CountingNoopRule { calls: Arc::clone(&calls), skip_if_unchanged: true, - })]) - .build(); - let ctx = SessionContext::new_with_state(state); + })], + ); let logical_plan = LogicalPlanBuilder::empty(false).build()?; ctx.state().create_physical_plan(&logical_plan).await?; ctx.state().create_physical_plan(&logical_plan).await?; @@ -3928,6 +3972,87 @@ mod tests { Ok(()) } + /// The optimizer only consults the rule it holds, so a rule wrapped for + /// timing or tracing decides for the rule inside it. Forwarding the answer + /// keeps the skip working; leaving the trait default in place turns it off + /// without any other visible effect, which is the trap this documents. + #[tokio::test] + async fn skip_unchanged_follows_what_a_wrapper_reports() -> Result<()> { + async fn wrapped_calls(forwards: bool) -> Result { + let calls = Arc::new(AtomicUsize::new(0)); + let rule = || { + Arc::new(WrappingRule { + inner: Arc::new(CountingNoopRule { + calls: Arc::clone(&calls), + skip_if_unchanged: true, + }), + forwards_skip_if_unchanged: forwards, + }) as Arc + }; + let ctx = session_with_rules(true, vec![rule(), rule()]); + let logical_plan = LogicalPlanBuilder::empty(false).build()?; + ctx.state().create_physical_plan(&logical_plan).await?; + Ok(calls.load(AtomicOrdering::Relaxed)) + } + + assert_eq!(wrapped_calls(true).await?, SKIPPED_CALLS); + assert_eq!(wrapped_calls(false).await?, 2); + Ok(()) + } + + /// Plans a query through the built-in rule list with two further + /// enforcement passes appended, the shape that motivates this feature, and + /// checks that turning the optimization on changes how often a rule runs + /// and nothing else: the plan itself must come out identical. + #[tokio::test] + async fn skip_unchanged_leaves_the_plan_alone() -> Result<()> { + async fn plan_with(skip_enabled: bool) -> Result { + let mut rules = PhysicalOptimizer::default().rules; + // Two trailing passes, as a downstream list has after inserting + // rewrites of its own behind the built-in enforcement. + rules.push(Arc::new(EnsureRequirements::new())); + rules.push(Arc::new(EnsureRequirements::new())); + let ctx = session_with_rules(skip_enabled, rules); + ctx.sql("CREATE TABLE t(a INT, b INT) AS VALUES (1, 10), (2, 20), (1, 30)") + .await? + .collect() + .await?; + let df = ctx + .sql("SELECT a, sum(b) FROM t GROUP BY a ORDER BY a") + .await?; + let plan = df.create_physical_plan().await?; + Ok(displayable(plan.as_ref()).indent(true).to_string()) + } + + assert_eq!(plan_with(true).await?, plan_with(false).await?); + Ok(()) + } + + /// `EXPLAIN VERBOSE` renders one plan snapshot per rule from the observer + /// callback, so a skipped rule still has to report the plan it would have + /// returned. Otherwise enabling the optimization would silently shorten + /// the explain output. + #[tokio::test] + async fn skip_unchanged_still_reports_every_rule_to_the_observer() -> Result<()> { + async fn explain_with(skip_enabled: bool) -> Result { + let rule = || { + Arc::new(CountingNoopRule { + calls: Arc::new(AtomicUsize::new(0)), + skip_if_unchanged: true, + }) as Arc + }; + let ctx = session_with_rules(skip_enabled, vec![rule(), rule()]); + let batches = ctx.sql("EXPLAIN VERBOSE SELECT 1").await?.collect().await?; + Ok(arrow::util::pretty::pretty_format_batches(&batches)?.to_string()) + } + + let skipped = explain_with(true).await?; + assert_eq!(skipped, explain_with(false).await?); + // Both passes are reported, including the one that did not run. + assert_eq!(skipped.matches("counting_noop_rule").count(), 2); + Ok(()) + } + async fn aggregate_explain(logical_plan: &LogicalPlan) -> Result { let physical_plan = plan(logical_plan).await?; Ok(displayable(physical_plan.as_ref()).indent(true).to_string()) diff --git a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs index 493b912be7e19..b827bc791c030 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs @@ -261,10 +261,14 @@ impl PhysicalOptimizerRule for EnsureRequirements { } /// This rule derives everything it does from the plan and the config, and - /// the config is fixed for an optimization run, so re-running it on a plan - /// it just produced cannot change anything. Rule lists that enforce - /// requirements again after their own rewrites can therefore skip the - /// passes whose input nothing has touched. + /// the config is fixed for an optimization run, so run again on a plan it + /// just produced it can only arrive at that same plan. Rule lists that + /// enforce requirements again after their own rewrites can therefore skip + /// the passes whose input nothing has touched. + /// + /// Note the rule rebuilds the tree rather than returning its input, so the + /// repeated pass it saves is a full reconstruction of the plan, not just + /// the analysis that decides nothing needs to change. fn skip_if_unchanged(&self) -> bool { true } diff --git a/datafusion/session/src/physical_optimizer.rs b/datafusion/session/src/physical_optimizer.rs index 6d35e1fd870c7..47829b9465aae 100644 --- a/datafusion/session/src/physical_optimizer.rs +++ b/datafusion/session/src/physical_optimizer.rs @@ -76,15 +76,22 @@ pub trait PhysicalOptimizerRule: Debug + std::any::Any { /// A human readable name for this optimizer rule fn name(&self) -> &str; - /// Whether this rule is a pure function of the plan it is given, so that - /// running it again on a plan object it previously returned cannot change - /// anything. + /// Whether running this rule again on a plan it produced itself is + /// guaranteed to describe the same plan. /// /// When a rule opts in, the optimizer remembers the plan the rule last - /// returned and skips the call when handed back that exact object. Plans - /// are reference counted and a rule returns its input untouched when it - /// has nothing to do, so this is an exact, allocation-free signal — no - /// hashing or structural comparison is involved. + /// returned and skips the call when handed back that exact object. The + /// test is pointer identity, which is exact and costs nothing: plans are + /// reference counted, and a rule that finds nothing to do returns its + /// input untouched, so an undisturbed stretch of the rule list carries the + /// same object through to the next pass. + /// + /// What is being claimed is idempotence, not identity. A rule may rebuild + /// the tree and hand back a fresh object every time — `EnsureRequirements` + /// does exactly that — and still qualify, because all the skip relies on + /// is that the second pass would arrive at the same plan as the first. + /// That is also what makes the skip worth having: the pass it removes + /// would have rebuilt the entire tree to end up back where it started. /// /// This is off by default and only consulted when /// `datafusion.optimizer.skip_unchanged_physical_rules` is enabled. It @@ -95,8 +102,9 @@ pub trait PhysicalOptimizerRule: Debug + std::any::Any { /// Leave this `false` for any rule whose output depends on state outside /// the plan — session state that can change between invocations, /// counters, randomness — since the same input would no longer imply the - /// same output. Debug builds verify the claim: when a skip would fire, - /// the rule is run anyway and the result is asserted to be unchanged. + /// same output. Debug builds verify the claim: where a skip would fire the + /// rule is run anyway, and the plan it returns is asserted to match the + /// one that was kept. /// /// The optimizer reads this from the rule it holds, so a rule that runs /// *other* rules inside its own [`optimize`] must forward their answer — diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 9a46837e9ec23..98495f3339d0b 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -147,7 +147,7 @@ The following configuration settings are available: | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | | datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | | datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.skip_unchanged_physical_rules | false | When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`] is skipped when its input is the very plan it returned last time, since re-running a pure rule on its own output cannot change anything. The default rule list has no repeated rules, so this matters for custom rule lists (`with_physical_optimizer_rules`) that enforce requirements again after their own rewrites. [`PhysicalOptimizerRule::skip_if_unchanged`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged | +| datafusion.optimizer.skip_unchanged_physical_rules | false | When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) is skipped when the plan handed to it is the very plan it returned last time, since an idempotent rule run on its own output only arrives at that same plan again. "Nothing changed" is tested by pointer identity, which is exact and free: plans are reference counted, and a rule that finds nothing to do returns its input untouched, so an undisturbed stretch of the rule list carries the same object through to the next pass. Physical rules run as a fixed sequence with no fixpoint loop, so this matters for rule lists that hold the same rule more than once. The built-in list holds none twice, which is why this is off by default; it is aimed at lists installed through `SessionStateBuilder::with_physical_optimizer_rules`, where custom rewrites are inserted after the built-in requirement enforcement and each of them needs requirements enforced again. | | datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | | datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | | datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | From 98e0e3fe5cd4d836791ce5e6e77db7954ee37112 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 12:08:17 +0800 Subject: [PATCH 08/11] refactor: name skippable rules in config instead of on the trait Replaces the `PhysicalOptimizerRule::skip_if_unchanged` opt-in with a list of rule names in the config option, which now holds names rather than a bool. The opt-in was a trait method that exactly one built-in rule set. An audit of all 21 built-in rules, added here as a test, finds every one of them idempotent, so singling out EnsureRequirements was arbitrary and opting in all 21 would be 21 unverified claims plus a decision to make for each new rule. Whether a chain repeats a rule is a property of the chain, not of the rule, so the chain's owner is who can say it. Naming rules also removes a trap the trait version carried. The optimizer only asks the outermost rule, so a rule wrapped for timing or tracing had to forward the answer, and one that left the default in place silently turned the optimization off. Names need no such cooperation: a wrapper already reports the name it wraps, because that is what EXPLAIN VERBOSE shows. It costs a semver-visible trait method and gains reach over rules the caller does not own. The debug self-check still verifies every skip. Tests: skips a repeated pass; inert when the name is absent, unknown or misspelt; reads a list with the spacing people write; handles an interleaved enforce/rewrite chain; does not leak between plans; follows the name a wrapper reports and not the wrapper's own; leaves a corpus of nine plans byte-identical through the built-in chain plus two trailing EnsureRequirements passes; keeps EXPLAIN VERBOSE output unchanged; and holds every built-in rule to the idempotence the config asserts. --- datafusion/common/src/config.rs | 32 +- datafusion/core/src/physical_planner.rs | 278 ++++++++++++------ .../src/ensure_requirements/mod.rs | 13 - datafusion/session/src/physical_optimizer.rs | 47 --- .../test_files/information_schema.slt | 4 +- docs/source/user-guide/configs.md | 2 +- 6 files changed, 198 insertions(+), 178 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 969948bd2a574..31714bf51dc32 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1597,24 +1597,22 @@ config_namespace! { /// repartitioning to increase parallelism to leverage more CPU cores pub enable_round_robin_repartition: bool, default = true - /// When set to true, a physical optimizer rule that opts in via - /// [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) - /// is skipped when the plan handed to it is the very plan it returned - /// last time, since an idempotent rule run on its own output only - /// arrives at that same plan again. "Nothing changed" is tested by - /// pointer identity, which is exact and free: plans are reference - /// counted, and a rule that finds nothing to do returns its input - /// untouched, so an undisturbed stretch of the rule list carries the - /// same object through to the next pass. + /// Comma separated names of physical optimizer rules that may be + /// skipped when the plan handed to them is the very plan they returned + /// last time. Empty, the default, disables the optimization. /// - /// Physical rules run as a fixed sequence with no fixpoint loop, so - /// this matters for rule lists that hold the same rule more than once. - /// The built-in list holds none twice, which is why this is off by - /// default; it is aimed at lists installed through - /// `SessionStateBuilder::with_physical_optimizer_rules`, where custom - /// rewrites are inserted after the built-in requirement enforcement - /// and each of them needs requirements enforced again. - pub skip_unchanged_physical_rules: bool, default = false + /// Physical rules run as a fixed sequence with no fixpoint loop, so a + /// list holding the same rule twice runs it again on a plan nothing + /// has touched since it produced it. Naming that rule lets the repeat + /// be skipped. "Nothing changed" is tested by pointer identity, which + /// is exact and costs nothing. + /// + /// Only name idempotent rules, meaning ones that run on their own + /// output arrive at the same plan again. Debug builds verify that + /// rather than trusting it. Names are matched against what a rule + /// reports as its name, which is what `EXPLAIN VERBOSE` shows; a name + /// matching no rule is ignored. + pub skip_unchanged_physical_rules: String, default = "".to_string() /// When set to true, the optimizer will attempt to perform limit operations /// during aggregations, if possible diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index f081d63c10940..79284eeab4957 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2965,23 +2965,32 @@ impl DefaultPhysicalPlanner { let optimizer_context = SessionOptimizerContext { session: session_state, }; - // Remembers the plan each opted-in rule last returned, so the rule can - // be skipped when handed back that exact object. Keyed by rule name - // rather than position, because a repeated rule is normally a second - // instance rather than the same one, and it is the rule's identity - // that makes re-running it pointless. Scoped to this call: rule - // instances are shared between queries, so this must not live on the - // rule itself. - let mut last_outputs = session_state + // The rules `skip_unchanged_physical_rules` names, paired with the plan + // each of them last returned, so a named rule can be skipped when + // handed back that exact object. + // + // The memo is keyed by rule name rather than by position, because a + // repeated rule is normally a second instance rather than the same + // one, and it is the rule's identity that makes re-running it + // pointless. Both halves are built only when the config names + // something, and both are scoped to this call: rule instances are + // shared between queries, so neither may live on the rule itself. + let configured = &session_state .config_options() .optimizer - .skip_unchanged_physical_rules - .then(HashMap::<&str, Arc>::new); + .skip_unchanged_physical_rules; + let mut skippable = (!configured.is_empty()).then(|| { + let names: HashSet<&str> = configured + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + .collect(); + (names, HashMap::<&str, Arc>::new()) + }); for optimizer in optimizers { - if let Some(memo) = last_outputs.as_ref() - && optimizer.skip_if_unchanged() - && let Some(last) = memo.get(optimizer.name()) + if let Some((_, last_outputs)) = skippable.as_ref() + && let Some(last) = last_outputs.get(optimizer.name()) && Arc::ptr_eq(last, &new_plan) { // The rule produced this exact plan and nothing since has @@ -3004,9 +3013,11 @@ impl DefaultPhysicalPlanner { debug_assert_eq!( displayable(rerun.as_ref()).indent(true).to_string(), displayable(new_plan.as_ref()).indent(true).to_string(), - "PhysicalOptimizer rule '{}' declares skip_if_unchanged() \ - but running it on a plan it had already produced changed \ - that plan, so the rule is not idempotent", + "PhysicalOptimizer rule '{}' is named in \ + datafusion.optimizer.skip_unchanged_physical_rules but \ + running it on a plan it had already produced changed that \ + plan, so the rule is not idempotent and must not be named \ + there", optimizer.name(), ); } @@ -3020,10 +3031,10 @@ impl DefaultPhysicalPlanner { .map_err(|e| { DataFusionError::Context(optimizer.name().to_string(), Box::new(e)) })?; - if let Some(memo) = last_outputs.as_mut() - && optimizer.skip_if_unchanged() + if let Some((names, last_outputs)) = skippable.as_mut() + && names.contains(optimizer.name()) { - memo.insert(optimizer.name(), Arc::clone(&new_plan)); + last_outputs.insert(optimizer.name(), Arc::clone(&new_plan)); } // This only checks the schema in release build, and performs additional checks in debug mode. @@ -3754,7 +3765,6 @@ mod tests { #[derive(Debug)] struct CountingNoopRule { calls: Arc, - skip_if_unchanged: bool, } impl PhysicalOptimizerRule for CountingNoopRule { @@ -3771,23 +3781,19 @@ mod tests { "counting_noop_rule" } - fn skip_if_unchanged(&self) -> bool { - self.skip_if_unchanged - } - fn schema_check(&self) -> bool { true } } /// Stands in for the instrumentation wrappers downstream projects put - /// around every rule to time or trace it. The optimizer only ever asks the - /// outermost rule, so a wrapper has to pass the answer through; one that - /// does not silently opts the rule it wraps back out. + /// around every rule to time or trace it. Naming rules by name is what + /// lets such a wrapper keep working, since it already has to report the + /// name it wraps for `EXPLAIN VERBOSE` to stay readable. #[derive(Debug)] struct WrappingRule { inner: Arc, - forwards_skip_if_unchanged: bool, + reports_inner_name: bool, } impl PhysicalOptimizerRule for WrappingRule { @@ -3808,11 +3814,11 @@ mod tests { } fn name(&self) -> &str { - self.inner.name() - } - - fn skip_if_unchanged(&self) -> bool { - self.forwards_skip_if_unchanged && self.inner.skip_if_unchanged() + if self.reports_inner_name { + self.inner.name() + } else { + "wrapping_rule" + } } fn schema_check(&self) -> bool { @@ -3820,20 +3826,22 @@ mod tests { } } - /// Expected invocations of a rule listed twice. Debug builds verify the - /// idempotence claim by running a skipped rule anyway and asserting it - /// changed nothing, so the call still happens there — what the skip saves - /// in debug is nothing, and in release it is the whole second pass. + /// Expected invocations of a rule listed twice and named in the config. + /// Debug builds verify the idempotence claim by running a skipped rule + /// anyway and asserting it changed nothing, so the call still happens + /// there: what the skip saves in debug is nothing, and in release it is + /// the whole second pass. const SKIPPED_CALLS: usize = if cfg!(debug_assertions) { 2 } else { 1 }; - /// A context whose physical rule list is exactly `rules`, with the skip - /// optimization on or off. + /// A context whose physical rule list is exactly `rules`, with the given + /// value for `skip_unchanged_physical_rules`. fn session_with_rules( - skip_enabled: bool, + skip_config: &str, rules: Vec>, ) -> SessionContext { let mut config = SessionConfig::new(); - config.options_mut().optimizer.skip_unchanged_physical_rules = skip_enabled; + config.options_mut().optimizer.skip_unchanged_physical_rules = + skip_config.to_string(); let state = SessionStateBuilder::new() .with_config(config) .with_default_features() @@ -3845,38 +3853,52 @@ mod tests { /// Plans the same rule twice, which is the shape a custom rule list takes /// when a rewrite between the two passes may or may not fire, and reports /// how many times the rule was actually asked to optimize. - async fn run_repeated_rule( - skip_if_unchanged: bool, - skip_enabled: bool, - ) -> Result { + async fn run_repeated_rule(skip_config: &str) -> Result { let calls = Arc::new(AtomicUsize::new(0)); let rule = || { Arc::new(CountingNoopRule { calls: Arc::clone(&calls), - skip_if_unchanged, }) as Arc }; - let ctx = session_with_rules(skip_enabled, vec![rule(), rule()]); + let ctx = session_with_rules(skip_config, vec![rule(), rule()]); let logical_plan = LogicalPlanBuilder::empty(false).build()?; ctx.state().create_physical_plan(&logical_plan).await?; Ok(calls.load(AtomicOrdering::Relaxed)) } - /// A rule that opted in is called once instead of twice: the second entry - /// receives the exact plan the first returned. + /// A named rule is called once instead of twice: the second entry receives + /// the exact plan the first returned. #[tokio::test] async fn skip_unchanged_skips_the_repeated_pass() -> Result<()> { - assert_eq!(run_repeated_rule(true, true).await?, SKIPPED_CALLS); + assert_eq!( + run_repeated_rule("counting_noop_rule").await?, + SKIPPED_CALLS + ); Ok(()) } - /// Off by default, and opting in without the config flag changes nothing: - /// both entries run, exactly as before this feature existed. + /// Off unless the rule is named, so an empty config behaves exactly as + /// before this feature existed, and a name that matches nothing, whether a + /// typo or a rule that is not in this list, is simply inert. #[tokio::test] - async fn skip_unchanged_is_inert_unless_both_sides_agree() -> Result<()> { - assert_eq!(run_repeated_rule(false, true).await?, 2); - assert_eq!(run_repeated_rule(true, false).await?, 2); - assert_eq!(run_repeated_rule(false, false).await?, 2); + async fn skip_unchanged_is_inert_unless_the_rule_is_named() -> Result<()> { + assert_eq!(run_repeated_rule("").await?, 2); + assert_eq!(run_repeated_rule("some_other_rule").await?, 2); + assert_eq!(run_repeated_rule("counting_noop_rul").await?, 2); + Ok(()) + } + + /// The config is a list, and reading it tolerates the spacing people + /// actually write. + #[tokio::test] + async fn skip_unchanged_reads_a_list_of_names() -> Result<()> { + for config in [ + "counting_noop_rule,some_other_rule", + "some_other_rule, counting_noop_rule", + " counting_noop_rule ,, ", + ] { + assert_eq!(run_repeated_rule(config).await?, SKIPPED_CALLS, "{config}"); + } Ok(()) } @@ -3923,7 +3945,6 @@ mod tests { let enforce = || { Arc::new(CountingNoopRule { calls: Arc::clone(&calls), - skip_if_unchanged: true, }) as Arc }; let rewrite = |name, rewrite| { @@ -3932,7 +3953,7 @@ mod tests { }; let ctx = session_with_rules( - true, + "counting_noop_rule", vec![ enforce(), // runs: nothing memoized yet rewrite("rewrite_that_fires", true), @@ -3957,10 +3978,9 @@ mod tests { async fn skip_unchanged_does_not_leak_between_plans() -> Result<()> { let calls = Arc::new(AtomicUsize::new(0)); let ctx = session_with_rules( - true, + "counting_noop_rule", vec![Arc::new(CountingNoopRule { calls: Arc::clone(&calls), - skip_if_unchanged: true, })], ); let logical_plan = LogicalPlanBuilder::empty(false).build()?; @@ -3972,59 +3992,122 @@ mod tests { Ok(()) } - /// The optimizer only consults the rule it holds, so a rule wrapped for - /// timing or tracing decides for the rule inside it. Forwarding the answer - /// keeps the skip working; leaving the trait default in place turns it off - /// without any other visible effect, which is the trap this documents. + /// Rules are matched by the name they report, so a rule wrapped for timing + /// or tracing is reached through the name the wrapper passes through, with + /// no cooperation needed from the wrapper beyond what `EXPLAIN VERBOSE` + /// already requires of it. A wrapper that renames what it wraps is + /// addressed by its own name instead. #[tokio::test] - async fn skip_unchanged_follows_what_a_wrapper_reports() -> Result<()> { - async fn wrapped_calls(forwards: bool) -> Result { + async fn skip_unchanged_follows_the_name_a_wrapper_reports() -> Result<()> { + async fn wrapped_calls( + reports_inner_name: bool, + skip_config: &str, + ) -> Result { let calls = Arc::new(AtomicUsize::new(0)); let rule = || { Arc::new(WrappingRule { inner: Arc::new(CountingNoopRule { calls: Arc::clone(&calls), - skip_if_unchanged: true, }), - forwards_skip_if_unchanged: forwards, + reports_inner_name, }) as Arc }; - let ctx = session_with_rules(true, vec![rule(), rule()]); + let ctx = session_with_rules(skip_config, vec![rule(), rule()]); let logical_plan = LogicalPlanBuilder::empty(false).build()?; ctx.state().create_physical_plan(&logical_plan).await?; Ok(calls.load(AtomicOrdering::Relaxed)) } - assert_eq!(wrapped_calls(true).await?, SKIPPED_CALLS); - assert_eq!(wrapped_calls(false).await?, 2); + assert_eq!( + wrapped_calls(true, "counting_noop_rule").await?, + SKIPPED_CALLS + ); + assert_eq!(wrapped_calls(false, "counting_noop_rule").await?, 2); + assert_eq!(wrapped_calls(false, "wrapping_rule").await?, SKIPPED_CALLS); Ok(()) } - /// Plans a query through the built-in rule list with two further - /// enforcement passes appended, the shape that motivates this feature, and - /// checks that turning the optimization on changes how often a rule runs - /// and nothing else: the plan itself must come out identical. + /// Queries chosen to reach the operators the built-in rules act on. + const PLAN_CORPUS: &[&str] = &[ + "SELECT a, sum(b) FROM t GROUP BY a ORDER BY a", + "SELECT count(*) FROM t", + "SELECT DISTINCT a FROM t", + "SELECT * FROM t ORDER BY b LIMIT 5", + "SELECT a FROM t WHERE b > 10 ORDER BY a LIMIT 3", + "SELECT t.a, u.d FROM t JOIN u ON t.a = u.c", + "SELECT a, row_number() OVER (PARTITION BY a ORDER BY b) FROM t", + "SELECT a, b FROM t UNION ALL SELECT c, d FROM u", + "SELECT a, sum(b) FROM t GROUP BY a HAVING sum(b) > 5 ORDER BY a LIMIT 2", + ]; + + /// Plans every query in [`PLAN_CORPUS`] through `rules`. + async fn corpus_plans( + skip_config: &str, + rules: Vec>, + ) -> Result> { + let ctx = session_with_rules(skip_config, rules); + ctx.sql("CREATE TABLE t(a INT, b INT) AS VALUES (1,10),(2,20),(1,30)") + .await? + .collect() + .await?; + ctx.sql("CREATE TABLE u(c INT, d INT) AS VALUES (1,100),(3,300)") + .await? + .collect() + .await?; + let mut plans = Vec::with_capacity(PLAN_CORPUS.len()); + for query in PLAN_CORPUS { + let plan = ctx.sql(query).await?.create_physical_plan().await?; + plans.push(displayable(plan.as_ref()).indent(true).to_string()); + } + Ok(plans) + } + + /// The built-in list with two further enforcement passes appended, as a + /// downstream list has after inserting rewrites of its own behind the + /// built-in enforcement. This is the shape that motivates the feature. + fn rules_with_trailing_enforcement() + -> Vec> { + let mut rules = PhysicalOptimizer::default().rules; + rules.push(Arc::new(EnsureRequirements::new())); + rules.push(Arc::new(EnsureRequirements::new())); + rules + } + + /// Turning the optimization on must change how often a rule runs and + /// nothing else, so every plan in the corpus has to come out identical. #[tokio::test] async fn skip_unchanged_leaves_the_plan_alone() -> Result<()> { - async fn plan_with(skip_enabled: bool) -> Result { - let mut rules = PhysicalOptimizer::default().rules; - // Two trailing passes, as a downstream list has after inserting - // rewrites of its own behind the built-in enforcement. - rules.push(Arc::new(EnsureRequirements::new())); - rules.push(Arc::new(EnsureRequirements::new())); - let ctx = session_with_rules(skip_enabled, rules); - ctx.sql("CREATE TABLE t(a INT, b INT) AS VALUES (1, 10), (2, 20), (1, 30)") - .await? - .collect() - .await?; - let df = ctx - .sql("SELECT a, sum(b) FROM t GROUP BY a ORDER BY a") - .await?; - let plan = df.create_physical_plan().await?; - Ok(displayable(plan.as_ref()).indent(true).to_string()) - } + let skipped = + corpus_plans("EnsureRequirements", rules_with_trailing_enforcement()).await?; + let stock = corpus_plans("", rules_with_trailing_enforcement()).await?; + assert_eq!(skipped, stock); + Ok(()) + } - assert_eq!(plan_with(true).await?, plan_with(false).await?); + /// Naming a rule in the config asserts that running it on its own output + /// arrives at the same plan. This checks that claim for every built-in + /// rule by running each one twice in place and requiring the corpus to + /// plan identically, and so records which of them may be named. + /// + /// All of them can, today. A rule that stops being idempotent breaks the + /// promise for anyone who named it, which is what this guards. + #[tokio::test] + async fn builtin_rules_are_idempotent() -> Result<()> { + let stock = PhysicalOptimizer::default().rules; + let baseline = corpus_plans("", stock.clone()).await?; + + for (position, rule) in stock.iter().enumerate() { + let mut doubled = stock.clone(); + doubled.insert(position + 1, Arc::clone(rule)); + assert_eq!( + corpus_plans("", doubled).await?, + baseline, + "running '{}' (position {position}) twice changed the plan, so it \ + is not idempotent and must not be named in \ + datafusion.optimizer.skip_unchanged_physical_rules", + rule.name(), + ); + } Ok(()) } @@ -4034,20 +4117,19 @@ mod tests { /// the explain output. #[tokio::test] async fn skip_unchanged_still_reports_every_rule_to_the_observer() -> Result<()> { - async fn explain_with(skip_enabled: bool) -> Result { + async fn explain_with(skip_config: &str) -> Result { let rule = || { Arc::new(CountingNoopRule { calls: Arc::new(AtomicUsize::new(0)), - skip_if_unchanged: true, }) as Arc }; - let ctx = session_with_rules(skip_enabled, vec![rule(), rule()]); + let ctx = session_with_rules(skip_config, vec![rule(), rule()]); let batches = ctx.sql("EXPLAIN VERBOSE SELECT 1").await?.collect().await?; Ok(arrow::util::pretty::pretty_format_batches(&batches)?.to_string()) } - let skipped = explain_with(true).await?; - assert_eq!(skipped, explain_with(false).await?); + let skipped = explain_with("counting_noop_rule").await?; + assert_eq!(skipped, explain_with("").await?); // Both passes are reported, including the one that did not run. assert_eq!(skipped.matches("counting_noop_rule").count(), 2); Ok(()) diff --git a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs index b827bc791c030..2bc57915b2315 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/mod.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/mod.rs @@ -260,19 +260,6 @@ impl PhysicalOptimizerRule for EnsureRequirements { "EnsureRequirements" } - /// This rule derives everything it does from the plan and the config, and - /// the config is fixed for an optimization run, so run again on a plan it - /// just produced it can only arrive at that same plan. Rule lists that - /// enforce requirements again after their own rewrites can therefore skip - /// the passes whose input nothing has touched. - /// - /// Note the rule rebuilds the tree rather than returning its input, so the - /// repeated pass it saves is a full reconstruction of the plan, not just - /// the analysis that decides nothing needs to change. - fn skip_if_unchanged(&self) -> bool { - true - } - fn schema_check(&self) -> bool { true } diff --git a/datafusion/session/src/physical_optimizer.rs b/datafusion/session/src/physical_optimizer.rs index 47829b9465aae..751a8e12d93ed 100644 --- a/datafusion/session/src/physical_optimizer.rs +++ b/datafusion/session/src/physical_optimizer.rs @@ -76,53 +76,6 @@ pub trait PhysicalOptimizerRule: Debug + std::any::Any { /// A human readable name for this optimizer rule fn name(&self) -> &str; - /// Whether running this rule again on a plan it produced itself is - /// guaranteed to describe the same plan. - /// - /// When a rule opts in, the optimizer remembers the plan the rule last - /// returned and skips the call when handed back that exact object. The - /// test is pointer identity, which is exact and costs nothing: plans are - /// reference counted, and a rule that finds nothing to do returns its - /// input untouched, so an undisturbed stretch of the rule list carries the - /// same object through to the next pass. - /// - /// What is being claimed is idempotence, not identity. A rule may rebuild - /// the tree and hand back a fresh object every time — `EnsureRequirements` - /// does exactly that — and still qualify, because all the skip relies on - /// is that the second pass would arrive at the same plan as the first. - /// That is also what makes the skip worth having: the pass it removes - /// would have rebuilt the entire tree to end up back where it started. - /// - /// This is off by default and only consulted when - /// `datafusion.optimizer.skip_unchanged_physical_rules` is enabled. It - /// pays off for rule lists that run the same rule more than once, which - /// is common when downstream rewrites are inserted after the built-in - /// requirement enforcement and each needs its requirements re-enforced. - /// - /// Leave this `false` for any rule whose output depends on state outside - /// the plan — session state that can change between invocations, - /// counters, randomness — since the same input would no longer imply the - /// same output. Debug builds verify the claim: where a skip would fire the - /// rule is run anyway, and the plan it returns is asserted to match the - /// one that was kept. - /// - /// The optimizer reads this from the rule it holds, so a rule that runs - /// *other* rules inside its own [`optimize`] must forward their answer — - /// in practice `all()` over the rules it wraps, since the wrapper is only - /// skippable if every rule it would have run is. A wrapper that leaves - /// this at the default silently opts its inner rules out: - /// - /// ```text - /// fn skip_if_unchanged(&self) -> bool { - /// self.wrapped.iter().all(|rule| rule.skip_if_unchanged()) - /// } - /// ``` - /// - /// [`optimize`]: PhysicalOptimizerRule::optimize - fn skip_if_unchanged(&self) -> bool { - false - } - /// A flag to indicate whether the physical planner should validate that the rule will not /// change the schema of the plan after the rewriting. /// Some of the optimization rules might change the nullable properties of the schema diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 9dff7abc4d049..baf73d22e8fb0 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -339,7 +339,7 @@ datafusion.optimizer.repartition_joins true datafusion.optimizer.repartition_sorts true datafusion.optimizer.repartition_windows true datafusion.optimizer.skip_failed_rules false -datafusion.optimizer.skip_unchanged_physical_rules false +datafusion.optimizer.skip_unchanged_physical_rules (empty) datafusion.optimizer.subset_repartition_threshold 4 datafusion.optimizer.top_down_join_key_reordering true datafusion.optimizer.use_statistics_registry false @@ -501,7 +501,7 @@ datafusion.optimizer.repartition_joins true Should DataFusion repartition data u datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail -datafusion.optimizer.skip_unchanged_physical_rules false When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) is skipped when the plan handed to it is the very plan it returned last time, since an idempotent rule run on its own output only arrives at that same plan again. "Nothing changed" is tested by pointer identity, which is exact and free: plans are reference counted, and a rule that finds nothing to do returns its input untouched, so an undisturbed stretch of the rule list carries the same object through to the next pass. Physical rules run as a fixed sequence with no fixpoint loop, so this matters for rule lists that hold the same rule more than once. The built-in list holds none twice, which is why this is off by default; it is aimed at lists installed through `SessionStateBuilder::with_physical_optimizer_rules`, where custom rewrites are inserted after the built-in requirement enforcement and each of them needs requirements enforced again. +datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys datafusion.optimizer.use_statistics_registry false (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 98495f3339d0b..3ea5ba45d0209 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -147,7 +147,7 @@ The following configuration settings are available: | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | | datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | | datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.skip_unchanged_physical_rules | false | When set to true, a physical optimizer rule that opts in via [`PhysicalOptimizerRule::skip_if_unchanged`](https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html#method.skip_if_unchanged) is skipped when the plan handed to it is the very plan it returned last time, since an idempotent rule run on its own output only arrives at that same plan again. "Nothing changed" is tested by pointer identity, which is exact and free: plans are reference counted, and a rule that finds nothing to do returns its input untouched, so an undisturbed stretch of the rule list carries the same object through to the next pass. Physical rules run as a fixed sequence with no fixpoint loop, so this matters for rule lists that hold the same rule more than once. The built-in list holds none twice, which is why this is off by default; it is aimed at lists installed through `SessionStateBuilder::with_physical_optimizer_rules`, where custom rewrites are inserted after the built-in requirement enforcement and each of them needs requirements enforced again. | +| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. | | datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | | datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | | datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | From 21994a5915051139c63c73e51534b5940fb2c54b Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 12:41:05 +0800 Subject: [PATCH 09/11] fix: warn that a config name stands for a behaviour, not an instance Every rule answering to a configured name shares one memo entry, so two rules that behave differently must not share a name. The built-in chain breaks that: OutputRequirements reports one name for the instance that adds requirements and the instance that removes them again, and naming it lets the first one's output suppress the second. The debug self-check catches it, but the config had claimed the built-in list holds no rule twice, which is wrong for OutputRequirements and ProjectionPushdown. Documents the constraint and pins the two repeated names in a test, so this is revisited if either rule is renamed. Giving OutputRequirements a distinct name per mode would make it nameable, and would disambiguate it in EXPLAIN VERBOSE too, but that is a separate change. --- datafusion/common/src/config.rs | 5 ++++ datafusion/core/src/physical_planner.rs | 26 +++++++++++++++++++ .../test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 31714bf51dc32..f4d90f0f6fa31 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1612,6 +1612,11 @@ config_namespace! { /// rather than trusting it. Names are matched against what a rule /// reports as its name, which is what `EXPLAIN VERBOSE` shows; a name /// matching no rule is ignored. + /// + /// A name has to identify a behaviour, because every rule answering to + /// it is treated as the same rule. The built-in `OutputRequirements` + /// reports one name for two instances that do opposite things, so it + /// must not be named here. pub skip_unchanged_physical_rules: String, default = "".to_string() /// When set to true, the optimizer will attempt to perform limit operations diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 79284eeab4957..8080ad314885a 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -4111,6 +4111,32 @@ mod tests { Ok(()) } + /// A name stands for a behaviour, since every rule answering to it shares + /// one memo entry. The built-in chain breaks that for `OutputRequirements`, + /// whose two instances add and then remove the same requirements under one + /// name, so naming it in the config would let the first instance's output + /// suppress the second. Pinned here because the config documents it, and + /// because giving the two modes distinct names would make it safe. + /// + /// `ProjectionPushdown` repeats too, but both instances are the same rule + /// doing the same thing, which is the case the feature is built for. + #[tokio::test] + async fn builtin_chain_repeats_two_rule_names() -> Result<()> { + let stock = PhysicalOptimizer::default().rules; + let mut counts = HashMap::<&str, usize>::new(); + for rule in &stock { + *counts.entry(rule.name()).or_default() += 1; + } + let mut repeated: Vec<&str> = counts + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(name, _)| name) + .collect(); + repeated.sort_unstable(); + assert_eq!(repeated, ["OutputRequirements", "ProjectionPushdown"]); + Ok(()) + } + /// `EXPLAIN VERBOSE` renders one plan snapshot per rule from the observer /// callback, so a skipped rule still has to report the plan it would have /// returned. Otherwise enabling the optimization would silently shorten diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index baf73d22e8fb0..54aed566e1ee8 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -501,7 +501,7 @@ datafusion.optimizer.repartition_joins true Should DataFusion repartition data u datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail -datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. +datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it is treated as the same rule. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys datafusion.optimizer.use_statistics_registry false (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 3ea5ba45d0209..a64c5a88c3725 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -147,7 +147,7 @@ The following configuration settings are available: | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | | datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | | datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. | +| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it is treated as the same rule. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. | | datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | | datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | | datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | From ba0b52cd64c8e90c6e8c2c8187e77f0fb98c756c Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 14:46:34 +0800 Subject: [PATCH 10/11] fix: skip on a proven fixpoint, not on the rule's last output The skip keyed on the plan a rule last returned, on the assumption that a rule handed back its own output has nothing left to do. That assumption is false: a rule is not required to reach its fixpoint in one pass, and EnsureRequirements routinely does not. Enabling it against a real chain tripped the debug self-check, with the second pass moving a RepartitionExec below a SortExec and switching the sort to per-partition, so the skipped pass would have silently cost that rewrite. Key on the plan the rule was *given* instead, and record it only after the rule has run and returned that same plan. A skip then replays an outcome already observed rather than predicting one, and a rule still converging records nothing and keeps running. Plans are compared by rendered form rather than by pointer. A rule that changes nothing still commonly rebuilds the tree, so pointer identity cannot see a fixpoint: on a real 34-node plan, of 23 passes that left the plan byte-identical only 7 also returned the input object. Collisions are handled by HashSet comparing on hit rather than trusting a hash. The config stays out of the key because what is recorded is scoped to one planning run, where it cannot change. Measured through a downstream chain that enforces requirements six times: physical optimization 208.9ms -> 147.8ms, planning wall 350.7ms -> 295.8ms, two passes skipped, EXPLAIN VERBOSE byte-identical. Adds a test for the case that makes the old key wrong: a rule needing several passes to converge must keep running, and the plan must come out as it does with the optimization off. --- datafusion/common/src/config.rs | 36 ++- datafusion/core/src/physical_planner.rs | 184 ++++++++--- .../test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 302 +++++++++--------- 4 files changed, 313 insertions(+), 211 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index f4d90f0f6fa31..45501b05cb96c 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1598,25 +1598,31 @@ config_namespace! { pub enable_round_robin_repartition: bool, default = true /// Comma separated names of physical optimizer rules that may be - /// skipped when the plan handed to them is the very plan they returned - /// last time. Empty, the default, disables the optimization. + /// skipped when handed a plan they have already been seen to leave + /// untouched. Empty, the default, disables the optimization. /// /// Physical rules run as a fixed sequence with no fixpoint loop, so a - /// list holding the same rule twice runs it again on a plan nothing - /// has touched since it produced it. Naming that rule lets the repeat - /// be skipped. "Nothing changed" is tested by pointer identity, which - /// is exact and costs nothing. + /// list holding the same rule several times runs it again on plans it + /// has already settled. Naming that rule lets those repeats be + /// answered from what was observed rather than re-derived. /// - /// Only name idempotent rules, meaning ones that run on their own - /// output arrive at the same plan again. Debug builds verify that - /// rather than trusting it. Names are matched against what a rule - /// reports as its name, which is what `EXPLAIN VERBOSE` shows; a name - /// matching no rule is ignored. + /// A plan is remembered only after the rule has run on it and returned + /// the same plan, so a skip replays an outcome already seen rather + /// than predicting one. A rule that has not yet reached its fixpoint + /// records nothing and keeps running, which matters because rules are + /// not required to settle in a single pass. /// - /// A name has to identify a behaviour, because every rule answering to - /// it is treated as the same rule. The built-in `OutputRequirements` - /// reports one name for two instances that do opposite things, so it - /// must not be named here. + /// What is remembered is scoped to one planning run, and plans are + /// compared by their rendered form, since a rule that changes nothing + /// still commonly rebuilds the tree. Debug builds re-run a skipped + /// rule and check it still leaves the plan alone. + /// + /// Names are matched against what a rule reports as its name, which is + /// what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A + /// name has to identify a behaviour, because every rule answering to it + /// shares one record. The built-in `OutputRequirements` reports one + /// name for two instances that do opposite things, so it must not be + /// named here. pub skip_unchanged_physical_rules: String, default = "".to_string() /// When set to true, the optimizer will attempt to perform limit operations diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 8080ad314885a..bc940e0d58416 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2965,64 +2965,82 @@ impl DefaultPhysicalPlanner { let optimizer_context = SessionOptimizerContext { session: session_state, }; - // The rules `skip_unchanged_physical_rules` names, paired with the plan - // each of them last returned, so a named rule can be skipped when - // handed back that exact object. + // For each rule `skip_unchanged_physical_rules` names, the plans that + // rule has been *observed* to leave untouched, so a later pass handed + // one of them can return it instead of re-deriving it. // - // The memo is keyed by rule name rather than by position, because a - // repeated rule is normally a second instance rather than the same - // one, and it is the rule's identity that makes re-running it - // pointless. Both halves are built only when the config names - // something, and both are scoped to this call: rule instances are - // shared between queries, so neither may live on the rule itself. + // A plan is recorded only after the rule has actually run on it and + // produced the same plan back, so nothing here is an assumption: a + // skip replays an outcome already seen. That matters because a rule + // is not required to reach its fixpoint in one pass, and the plan it + // returns is frequently not yet one. Keying on the plan the rule was + // *given* rather than on the plan it last *returned* is what keeps + // those two cases apart. + // + // Plans are keyed by rendered form rather than by pointer, because a + // rule that changes nothing still commonly rebuilds the tree and + // returns a fresh object. `HashSet` compares on collision, so + // two plans that hash alike are not confused for one another. + // + // Scoped to this call, which keeps the config out of the key: it + // cannot change midway through one optimization run. Rule instances + // are shared between queries, so this must not live on the rule. let configured = &session_state .config_options() .optimizer .skip_unchanged_physical_rules; - let mut skippable = (!configured.is_empty()).then(|| { + let mut fixpoints = (!configured.is_empty()).then(|| { let names: HashSet<&str> = configured .split(',') .map(str::trim) .filter(|name| !name.is_empty()) .collect(); - (names, HashMap::<&str, Arc>::new()) + (names, HashMap::<&str, HashSet>::new()) }); for optimizer in optimizers { - if let Some((_, last_outputs)) = skippable.as_ref() - && let Some(last) = last_outputs.get(optimizer.name()) - && Arc::ptr_eq(last, &new_plan) + // Rendered once per pass when the rule is named, and reused to + // record the outcome below. + let mut rendered_input = None; + if let Some((names, seen)) = fixpoints.as_ref() + && names.contains(optimizer.name()) { - // The rule produced this exact plan and nothing since has - // replaced it, so running it again cannot change anything. - // Debug builds verify that claim rather than trusting it. - #[cfg(debug_assertions)] + let before = displayable(new_plan.as_ref()).indent(true).to_string(); + if seen + .get(optimizer.name()) + .is_some_and(|plans| plans.contains(&before)) { - let rerun = optimizer - .optimize_with_context(Arc::clone(&new_plan), &optimizer_context) - .map_err(|e| { - DataFusionError::Context( - optimizer.name().to_string(), - Box::new(e), + // This rule has already run on this exact plan and left it + // alone, so running it again yields the same plan. Debug + // builds check that rather than trusting it. + #[cfg(debug_assertions)] + { + let rerun = optimizer + .optimize_with_context( + Arc::clone(&new_plan), + &optimizer_context, ) - })?; - // Rules routinely rebuild the tree even where they - // change nothing, so a re-run legitimately hands back a - // fresh object. What has to hold is that it describes the - // same plan. - debug_assert_eq!( - displayable(rerun.as_ref()).indent(true).to_string(), - displayable(new_plan.as_ref()).indent(true).to_string(), - "PhysicalOptimizer rule '{}' is named in \ - datafusion.optimizer.skip_unchanged_physical_rules but \ - running it on a plan it had already produced changed that \ - plan, so the rule is not idempotent and must not be named \ - there", - optimizer.name(), - ); + .map_err(|e| { + DataFusionError::Context( + optimizer.name().to_string(), + Box::new(e), + ) + })?; + debug_assert_eq!( + displayable(rerun.as_ref()).indent(true).to_string(), + before, + "PhysicalOptimizer rule '{}' is named in \ + datafusion.optimizer.skip_unchanged_physical_rules \ + but stopped leaving a plan it had left alone before \ + untouched, so it does not depend only on the plan \ + and the config", + optimizer.name(), + ); + } + observer(new_plan.as_ref(), optimizer.as_ref()); + continue; } - observer(new_plan.as_ref(), optimizer.as_ref()); - continue; + rendered_input = Some(before); } let before_schema = new_plan.schema(); @@ -3031,10 +3049,14 @@ impl DefaultPhysicalPlanner { .map_err(|e| { DataFusionError::Context(optimizer.name().to_string(), Box::new(e)) })?; - if let Some((names, last_outputs)) = skippable.as_mut() - && names.contains(optimizer.name()) + // Record a fixpoint only where the rule demonstrably produced + // the plan it was given. A rule still working towards its + // fixpoint records nothing, so its next pass is not skipped. + if let Some(before) = rendered_input + && let Some((_, seen)) = fixpoints.as_mut() + && displayable(new_plan.as_ref()).indent(true).to_string() == before { - last_outputs.insert(optimizer.name(), Arc::clone(&new_plan)); + seen.entry(optimizer.name()).or_default().insert(before); } // This only checks the schema in release build, and performs additional checks in debug mode. @@ -3474,6 +3496,7 @@ mod tests { use datafusion_physical_expr::EquivalenceProperties; use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements; use datafusion_physical_optimizer::optimizer::PhysicalOptimizer; + use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions}; use datafusion_session::QueryPlanner; @@ -4027,6 +4050,79 @@ mod tests { Ok(()) } + /// Changes the plan a fixed number of times and is a no-op after that, + /// standing in for a rule that needs several passes to converge. + /// `EnsureRequirements` is one: on real plans its distribution and sorting + /// phases can each still find work on a plan it produced itself. + #[derive(Debug)] + struct ConvergesAfter { + remaining: AtomicUsize, + calls: Arc, + } + + impl PhysicalOptimizerRule for ConvergesAfter { + fn optimize( + &self, + plan: Arc, + _config: &ConfigOptions, + ) -> Result> { + self.calls.fetch_add(1, AtomicOrdering::Relaxed); + if self.remaining.load(AtomicOrdering::Relaxed) == 0 { + return Ok(plan); + } + self.remaining.fetch_sub(1, AtomicOrdering::Relaxed); + Ok(Arc::new(CoalescePartitionsExec::new(plan))) + } + + fn name(&self) -> &str { + "counting_noop_rule" + } + + fn schema_check(&self) -> bool { + true + } + } + + /// The case that makes "skip what the rule last returned" wrong and this + /// design right: a rule still working towards its fixpoint must keep + /// running. Only a plan the rule has been seen to leave alone is recorded, + /// so the passes that still have work to do are never skipped, and the + /// plan comes out exactly as it does with the optimization off. + #[tokio::test] + async fn skip_unchanged_does_not_skip_a_rule_that_has_not_converged() -> Result<()> { + async fn run(skip_config: &str) -> Result<(usize, String)> { + let calls = Arc::new(AtomicUsize::new(0)); + // Shared across the five entries, as one rule instance repeated in + // a chain would be: it converges after two rewrites. + let rule = Arc::new(ConvergesAfter { + remaining: AtomicUsize::new(2), + calls: Arc::clone(&calls), + }) as Arc; + let ctx = session_with_rules( + skip_config, + (0..5).map(|_| Arc::clone(&rule)).collect(), + ); + let logical_plan = LogicalPlanBuilder::empty(false).build()?; + let plan = ctx.state().create_physical_plan(&logical_plan).await?; + Ok(( + calls.load(AtomicOrdering::Relaxed), + displayable(plan.as_ref()).indent(true).to_string(), + )) + } + + let (off_calls, off_plan) = run("").await?; + let (on_calls, on_plan) = run("counting_noop_rule").await?; + + // Five entries, all of which run with the optimization off. + assert_eq!(off_calls, 5); + // With it on, the two rewriting passes and the one that proves the + // fixpoint still run; only the last two are skipped. + assert_eq!(on_calls, 3 + 2 * (SKIPPED_CALLS - 1)); + // And the plan is unaffected, which is the point. + assert_eq!(on_plan, off_plan); + Ok(()) + } + /// Queries chosen to reach the operators the built-in rules act on. const PLAN_CORPUS: &[&str] = &[ "SELECT a, sum(b) FROM t GROUP BY a ORDER BY a", diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 54aed566e1ee8..0de6032988c76 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -501,7 +501,7 @@ datafusion.optimizer.repartition_joins true Should DataFusion repartition data u datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail -datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it is treated as the same rule. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. +datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. Naming that rule lets those repeats be answered from what was observed rather than re-derived. A plan is remembered only after the rule has run on it and returned the same plan, so a skip replays an outcome already seen rather than predicting one. A rule that has not yet reached its fixpoint records nothing and keeps running, which matters because rules are not required to settle in a single pass. What is remembered is scoped to one planning run, and plans are compared by their rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it still leaves the plan alone. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it shares one record. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys datafusion.optimizer.use_statistics_registry false (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index a64c5a88c3725..6618335d200d6 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -63,157 +63,157 @@ SET datafusion.execution.target_partitions = '1'; The following configuration settings are available: -| key | default | description | -| ----------------------------------------------------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | -| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | -| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | -| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | -| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | -| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | -| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | -| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | -| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | -| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone | -| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | -| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | -| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | -| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | -| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | -| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | -| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | -| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | -| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | -| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | -| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | -| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | -| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | -| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | -| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | -| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | -| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | -| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | -| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | -| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | -| datafusion.execution.parquet.created_by | datafusion version 55.1.0 | (writing) Sets "created by" property | -| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | -| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | -| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | -| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | -| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | -| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | -| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | -| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | -| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | -| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | -| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | -| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | -| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | -| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | -| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | -| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | -| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback coordinates per-chunk left state (visited bitmap and probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) build a separate coordinator per task and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | -| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | -| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | -| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | -| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | -| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | -| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | -| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | -| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | -| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | -| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | -| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | -| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | -| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | -| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | -| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | -| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | -| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | -| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when the plan handed to them is the very plan they returned last time. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule twice runs it again on a plan nothing has touched since it produced it. Naming that rule lets the repeat be skipped. "Nothing changed" is tested by pointer identity, which is exact and costs nothing. Only name idempotent rules, meaning ones that run on their own output arrive at the same plan again. Debug builds verify that rather than trusting it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it is treated as the same rule. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. | -| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | -| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | -| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | -| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | -| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | -| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | -| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | -| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | -| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | -| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | -| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | -| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | -| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | -| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | -| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | -| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | -| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | -| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | -| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | -| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | -| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | -| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | -| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | -| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | -| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | -| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | -| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | -| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | -| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | -| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | -| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | -| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | -| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | -| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | -| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | -| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | -| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | -| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | -| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | -| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | -| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | -| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | -| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | -| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | -| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | -| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | -| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | -| datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | -| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | -| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | -| datafusion.format.null | | Format string for nulls | -| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | -| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | -| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | -| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | -| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | -| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | -| datafusion.format.types_info | false | Show types in visual representation batches | -| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | +| key | default | description | +| ----------------------------------------------------------------------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | +| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | +| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | +| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | +| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | +| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | +| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | +| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | +| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | +| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone | +| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | +| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | +| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | +| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | +| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | +| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | +| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | +| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | +| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | +| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | +| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | +| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | +| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | +| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | +| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | +| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | +| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | +| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | +| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | +| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | +| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | +| datafusion.execution.parquet.created_by | datafusion version 55.1.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | +| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | +| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | +| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | +| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | +| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | +| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | +| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | +| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | +| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | +| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | +| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | +| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | +| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | +| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | +| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | +| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback coordinates per-chunk left state (visited bitmap and probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) build a separate coordinator per task and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | +| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | +| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | +| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | +| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | +| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | +| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | +| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | +| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | +| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | +| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | +| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | +| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | +| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | +| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | +| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | +| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | +| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | +| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | +| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | +| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. Naming that rule lets those repeats be answered from what was observed rather than re-derived. A plan is remembered only after the rule has run on it and returned the same plan, so a skip replays an outcome already seen rather than predicting one. A rule that has not yet reached its fixpoint records nothing and keeps running, which matters because rules are not required to settle in a single pass. What is remembered is scoped to one planning run, and plans are compared by their rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it still leaves the plan alone. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it shares one record. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. | +| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | +| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | +| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | +| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | +| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | +| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | +| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | +| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | +| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | +| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | +| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | +| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | +| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | +| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | +| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | +| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | +| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | +| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | +| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | +| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | +| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | +| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | +| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | +| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | +| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | +| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | +| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | +| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | +| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | +| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | +| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | +| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | +| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | +| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | +| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | +| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | +| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | +| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | +| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | +| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | +| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | +| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | +| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | +| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | +| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | +| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | +| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | +| datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | +| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | +| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | +| datafusion.format.null | | Format string for nulls | +| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | +| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | +| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | +| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | +| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | +| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | +| datafusion.format.types_info | false | Show types in visual representation batches | +| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | You can also reset configuration options to default settings via SQL using the `RESET` command. For example, to set and reset `datafusion.execution.batch_size`: From b38296b154ab85156e944134c15938d6bc6bf895 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 16 Sep 2026 14:55:29 +0800 Subject: [PATCH 11/11] docs: keep the config description inside the table's existing width The generated configs.md table pads every cell to the widest one, so a description longer than the current maximum reflows all 150 rows and buries the one row that was actually added. Trims it back under that width; the reasoning it carried is in the optimizer loop's comments and in the issue. --- datafusion/common/src/config.rs | 29 +- .../test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 302 +++++++++--------- 3 files changed, 163 insertions(+), 170 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 45501b05cb96c..9db7e7c66b0f9 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1603,26 +1603,19 @@ config_namespace! { /// /// Physical rules run as a fixed sequence with no fixpoint loop, so a /// list holding the same rule several times runs it again on plans it - /// has already settled. Naming that rule lets those repeats be - /// answered from what was observed rather than re-derived. - /// - /// A plan is remembered only after the rule has run on it and returned - /// the same plan, so a skip replays an outcome already seen rather - /// than predicting one. A rule that has not yet reached its fixpoint - /// records nothing and keeps running, which matters because rules are - /// not required to settle in a single pass. - /// - /// What is remembered is scoped to one planning run, and plans are - /// compared by their rendered form, since a rule that changes nothing - /// still commonly rebuilds the tree. Debug builds re-run a skipped - /// rule and check it still leaves the plan alone. + /// has already settled. A plan is remembered only after the rule ran + /// on it and returned that same plan, so a skip replays an observed + /// outcome rather than predicting one; a rule that has not yet + /// converged records nothing and keeps running. What is remembered is + /// scoped to one planning run, and plans are compared by rendered + /// form, since a rule that changes nothing still commonly rebuilds the + /// tree. Debug builds re-run a skipped rule and check it. /// /// Names are matched against what a rule reports as its name, which is - /// what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A - /// name has to identify a behaviour, because every rule answering to it - /// shares one record. The built-in `OutputRequirements` reports one - /// name for two instances that do opposite things, so it must not be - /// named here. + /// what `EXPLAIN VERBOSE` shows; an unmatched name is ignored. A name + /// stands for a behaviour, since every rule answering to it shares one + /// record: the built-in `OutputRequirements` names two instances that + /// do opposite things, so it must not be listed. pub skip_unchanged_physical_rules: String, default = "".to_string() /// When set to true, the optimizer will attempt to perform limit operations diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 0de6032988c76..b7fea64fbc568 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -501,7 +501,7 @@ datafusion.optimizer.repartition_joins true Should DataFusion repartition data u datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail -datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. Naming that rule lets those repeats be answered from what was observed rather than re-derived. A plan is remembered only after the rule has run on it and returned the same plan, so a skip replays an outcome already seen rather than predicting one. A rule that has not yet reached its fixpoint records nothing and keeps running, which matters because rules are not required to settle in a single pass. What is remembered is scoped to one planning run, and plans are compared by their rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it still leaves the plan alone. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it shares one record. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. +datafusion.optimizer.skip_unchanged_physical_rules (empty) Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. A plan is remembered only after the rule ran on it and returned that same plan, so a skip replays an observed outcome rather than predicting one; a rule that has not yet converged records nothing and keeps running. What is remembered is scoped to one planning run, and plans are compared by rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; an unmatched name is ignored. A name stands for a behaviour, since every rule answering to it shares one record: the built-in `OutputRequirements` names two instances that do opposite things, so it must not be listed. datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys datafusion.optimizer.use_statistics_registry false (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 6618335d200d6..01bb94c6685e0 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -63,157 +63,157 @@ SET datafusion.execution.target_partitions = '1'; The following configuration settings are available: -| key | default | description | -| ----------------------------------------------------------------------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | -| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | -| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | -| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | -| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | -| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | -| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | -| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | -| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | -| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone | -| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | -| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | -| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | -| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | -| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | -| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | -| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | -| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | -| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | -| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | -| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | -| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | -| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | -| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | -| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | -| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | -| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | -| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | -| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | -| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | -| datafusion.execution.parquet.created_by | datafusion version 55.1.0 | (writing) Sets "created by" property | -| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | -| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | -| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | -| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | -| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | -| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | -| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | -| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | -| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | -| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | -| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | -| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | -| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | -| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | -| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | -| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | -| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback coordinates per-chunk left state (visited bitmap and probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) build a separate coordinator per task and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | -| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | -| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | -| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | -| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | -| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | -| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | -| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | -| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | -| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | -| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | -| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | -| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | -| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | -| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | -| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | -| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | -| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | -| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. Naming that rule lets those repeats be answered from what was observed rather than re-derived. A plan is remembered only after the rule has run on it and returned the same plan, so a skip replays an outcome already seen rather than predicting one. A rule that has not yet reached its fixpoint records nothing and keeps running, which matters because rules are not required to settle in a single pass. What is remembered is scoped to one planning run, and plans are compared by their rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it still leaves the plan alone. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; a name matching no rule is ignored. A name has to identify a behaviour, because every rule answering to it shares one record. The built-in `OutputRequirements` reports one name for two instances that do opposite things, so it must not be named here. | -| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | -| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | -| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | -| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | -| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | -| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | -| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | -| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | -| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | -| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | -| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | -| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | -| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | -| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | -| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | -| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | -| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | -| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | -| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | -| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | -| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | -| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | -| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | -| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | -| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | -| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | -| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | -| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | -| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | -| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | -| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | -| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | -| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | -| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | -| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | -| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | -| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | -| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | -| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | -| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | -| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | -| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | -| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | -| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | -| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | -| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | -| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | -| datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | -| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | -| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | -| datafusion.format.null | | Format string for nulls | -| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | -| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | -| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | -| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | -| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | -| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | -| datafusion.format.types_info | false | Show types in visual representation batches | -| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | +| key | default | description | +| ----------------------------------------------------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | +| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | +| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | +| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | +| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | +| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | +| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | +| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | +| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | +| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone | +| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | +| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | +| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | +| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | +| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | +| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | +| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | +| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | +| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | +| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | +| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | +| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | +| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | +| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | +| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | +| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | +| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | +| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | +| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | +| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | +| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | +| datafusion.execution.parquet.created_by | datafusion version 55.1.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | +| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | +| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | +| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | +| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | +| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | +| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | +| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | +| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | +| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | +| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | +| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | +| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | +| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | +| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | +| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | +| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback coordinates per-chunk left state (visited bitmap and probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) build a separate coordinator per task and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | +| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | +| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | +| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | +| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | +| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | +| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | +| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | +| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | +| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | +| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | +| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | +| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | +| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | +| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | +| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | +| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | +| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | +| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | +| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | +| datafusion.optimizer.skip_unchanged_physical_rules | | Comma separated names of physical optimizer rules that may be skipped when handed a plan they have already been seen to leave untouched. Empty, the default, disables the optimization. Physical rules run as a fixed sequence with no fixpoint loop, so a list holding the same rule several times runs it again on plans it has already settled. A plan is remembered only after the rule ran on it and returned that same plan, so a skip replays an observed outcome rather than predicting one; a rule that has not yet converged records nothing and keeps running. What is remembered is scoped to one planning run, and plans are compared by rendered form, since a rule that changes nothing still commonly rebuilds the tree. Debug builds re-run a skipped rule and check it. Names are matched against what a rule reports as its name, which is what `EXPLAIN VERBOSE` shows; an unmatched name is ignored. A name stands for a behaviour, since every rule answering to it shares one record: the built-in `OutputRequirements` names two instances that do opposite things, so it must not be listed. | +| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | +| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | +| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | +| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | +| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | +| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | +| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | +| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | +| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | +| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | +| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | +| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | +| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | +| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | +| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | +| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | +| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | +| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | +| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | +| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | +| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | +| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | +| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | +| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | +| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | +| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | +| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | +| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | +| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | +| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | +| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | +| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | +| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | +| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | +| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | +| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | +| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | +| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | +| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | +| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | +| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | +| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | +| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | +| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | +| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | +| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | +| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | +| datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | +| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | +| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | +| datafusion.format.null | | Format string for nulls | +| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | +| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | +| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | +| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | +| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | +| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | +| datafusion.format.types_info | false | Show types in visual representation batches | +| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | You can also reset configuration options to default settings via SQL using the `RESET` command. For example, to set and reset `datafusion.execution.batch_size`: