From b2d72e635e1c5cca5aebe61492b0fdfb17e131b7 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 17 Aug 2026 16:22:56 +0800 Subject: [PATCH] branch-4.2 [opt](aggregate) eliminate FD-redundant group-by keys via ANY_VALUE wrapping (#64849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? When a group-by key is functionally dependent on another key (e.g. s_suppkey -> s_name via PK) but required in output, remove it from GROUP BY and wrap with ANY_VALUE(). Previously EliminateGroupByKey kept such keys in GROUP BY to preserve SQL semantics. Now they are replaced with ANY_VALUE wrappers in the output, allowing the group-by set to be minimized while keeping the column in SELECT. Public findCanBeRemovedExpressions() API preserved for backward compatibility. Internal logic split into FindResult with separate removeExpression and wrapWithAnyValue sets. Test: testEliminateByPkWithOutputNeeded verifies ANY_VALUE wrapping when SELECT contains an FD-redundant group-by key. Issue Number: close #xxx Related PR: #65982 #66801 #66803 上面 3 个 pr 是原有 master 的bug fix. pick 这个 pr 前, 确保上面 3 个 pr 已经 pick Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/job/extensions/mtmv/MTMVTask.java | 6 +- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 1 + .../doris/nereids/jobs/executor/Rewriter.java | 6 +- ...AbstractMaterializedViewAggregateRule.java | 4 +- .../mv/PreMaterializedViewRewriter.java | 1 + .../rules/expression/ExpressionRewrite.java | 4 +- .../rules/rewrite/EliminateGroupByKey.java | 239 +++++++++++++--- .../rewrite/SplitMultiDistinctStrategy.java | 2 +- .../plans/commands/info/CreateMTMVInfo.java | 2 +- .../mv/PreMaterializedViewRewriterTest.java | 15 +- .../mv/MaterializedViewUtilsTest.java | 2 +- .../EliminateGroupByKeyByUniformTest.java | 4 + .../rewrite/EliminateGroupByKeyTest.java | 134 ++++++++- .../eliminate_gby_key.groovy | 10 +- .../mv/agg_variety/agg_variety.groovy | 2 +- .../aggregate_without_roll_up.groovy | 6 +- .../range_date_datetrunc_part_up.groovy | 2 + .../with_lock/dml_rewrite_with_lock.groovy | 267 +++++++++--------- ...nner_join_list_str_increment_create.groovy | 2 +- ...er_join_range_date_increment_create.groovy | 2 +- ..._join_range_number_increment_create.groovy | 2 +- .../mv/nested_mtmv/nested_mtmv.groovy | 2 +- 22 files changed, 510 insertions(+), 205 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index ef84d6071ca20b..8db27ca453b4da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -272,7 +272,8 @@ public void run() throws JobException { try { executeWithRetry(execPartitionNames, tableWithPartKey, ctx); } catch (Exception e) { - LOG.error("Execution failed after retries: {}", e.getMessage()); + LOG.error("Execution failed after retries, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } completedPartitions.addAll(execPartitionNames); @@ -282,7 +283,8 @@ public void run() throws JobException { mtmv.getDatabase().getFullName(), mtmv.getName(), getTaskId()); } catch (Throwable e) { if (getStatus() == TaskStatus.RUNNING) { - LOG.warn("run task failed: {}", e.getMessage()); + LOG.warn("run task failed, mvName: {}, taskId: {}", + mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } else { // if status is not `RUNNING`,maybe the task was canceled, therefore, it is a normal situation diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 17801b7607296f..8b6e3aa264aba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -109,6 +109,7 @@ public class MTMVPlanUtil { RuleType.ELIMINATE_JOIN_BY_FK, RuleType.ELIMINATE_JOIN_BY_UK, RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, + RuleType.ELIMINATE_GROUP_BY_KEY, RuleType.ELIMINATE_GROUP_BY, RuleType.SALT_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 9167a0aed681dd..fae0b74bb076a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -673,7 +673,6 @@ public class Rewriter extends AbstractBatchJobExecutor { cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), - topDown(new EliminateGroupByKey()), topDown(new PushDownAggThroughJoinOnPkFk()), topDown(new PullUpJoinFromUnionAll()) ), @@ -909,6 +908,11 @@ private static List getWholeTreeRewriteJobs( ))); rewriteJobs.addAll(jobs(topic("convert outer join to anti", custom(RuleType.CONVERT_OUTER_JOIN_TO_ANTI, ConvertOuterJoinToAntiJoin::new)))); + rewriteJobs.addAll(jobs(topic("eliminate Aggregate according to fd items", + cascadesContext -> cascadesContext.rewritePlanContainsTypes(LogicalAggregate.class) + || cascadesContext.rewritePlanContainsTypes(LogicalJoin.class) + || cascadesContext.rewritePlanContainsTypes(LogicalUnion.class), + custom(RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new)))); rewriteJobs.addAll(jobs(topic("eliminate group by key by uniform", custom(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, EliminateGroupByKeyByUniform::new)))); if (needOrExpansion) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 7ab098cd002e41..8a04cb66eca5c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.properties.DataTrait; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewAggregateRule.AggregateExpressionRewriteContext.ExpressionRewriteMode; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; @@ -569,7 +570,8 @@ private static boolean isGroupByEqualsAfterGroupByEliminate(Set quer Plan rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, childContext -> { Rewriter.getCteChildrenRewriter(childContext, - ImmutableList.of(Rewriter.topDown(new EliminateGroupByKey()))).execute(); + ImmutableList.of(Rewriter.custom( + RuleType.ELIMINATE_GROUP_BY_KEY, EliminateGroupByKey::new))).execute(); return childContext.getRewritePlan(); }, viewProject, viewProject, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index d098c00e40ba67..477670ed7308bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -68,6 +68,7 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); + NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index d8d903384ba325..bd5ba0afec320a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -235,10 +235,10 @@ public Rule build() { List groupByExprs = agg.getGroupByExpressions(); ExpressionRewriteContext context = new ExpressionRewriteContext(agg, ctx.cascadesContext); List newGroupByExprs = rewriter.rewrite(groupByExprs, context); - + boolean groupByChanged = !newGroupByExprs.equals(groupByExprs); List outputExpressions = agg.getOutputExpressions(); RewriteResult result = rewriteAll(outputExpressions, rewriter, context); - if (!result.changed) { + if (!result.changed && !groupByChanged) { return agg; } return new LogicalAggregate<>(newGroupByExprs, result.result, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index 4e1b3117ab53ff..cff93b15b2b58b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -17,90 +17,230 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.FuncDeps; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; - /** * Eliminate group by key based on fd item information. * such as: * for a -> b, we can get: * group by a, b, c => group by a, c + * + * When a group-by key is FD-redundant but still needed in the output, + * it is wrapped with any_value() and assigned a fresh ExprId. + * Upper plan references are rewritten via ExprIdRewriter so that + * all ancestor nodes see the new ExprIds. */ -@DependsRules({EliminateGroupBy.class, ColumnPruning.class}) -public class EliminateGroupByKey implements RewriteRuleFactory { +public class EliminateGroupByKey extends DefaultPlanRewriter> implements CustomRewriter { + private ExprIdRewriter exprIdReplacer; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.containsType(Aggregate.class)) { + return plan; + } + Map replaceMap = new HashMap<>(); + ExprIdRewriter.ReplaceRule replaceRule = new ExprIdRewriter.ReplaceRule(replaceMap, false); + exprIdReplacer = new ExprIdRewriter(replaceRule, jobContext); + return plan.accept(this, replaceMap); + } + + @Override + public Plan visit(Plan plan, Map replaceMap) { + plan = visitChildren(this, plan, replaceMap); + plan = exprIdReplacer.rewriteExpr(plan, replaceMap); + return plan; + } + + @Override + public Plan visitLogicalProject(LogicalProject proj, Map replaceMap) { + proj = visitChildren(this, proj, replaceMap); + + // Find the Aggregate child, possibly through a Filter + Plan child = proj.child(0); + LogicalAggregate agg; + boolean hasFilter = child instanceof LogicalFilter; + if (hasFilter && child.child(0) instanceof LogicalAggregate) { + agg = (LogicalAggregate) child.child(0); + } else if (child instanceof LogicalAggregate) { + agg = (LogicalAggregate) child; + } else { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Don't transform if source repeat is present + if (agg.getSourceRepeat().isPresent()) { + return exprIdReplacer.rewriteExpr(proj, replaceMap); + } + + // Rewrite proj and the filter (if present) through the replaceMap accumulated + // by visitChildren, so that ExprId replacements from nested rewrites + // (e.g. inner aggregates) are reflected in the required-output slot set. + proj = (LogicalProject) exprIdReplacer.rewriteExpr(proj, replaceMap); + if (hasFilter) { + child = exprIdReplacer.rewriteExpr(child, replaceMap); + } + + // Compute requireOutput: slots needed by the Project (and Filter, if present) + Set requireOutput = new HashSet<>(proj.getInputSlots()); + if (hasFilter) { + requireOutput.addAll(child.getInputSlots()); + } + + // Transform the aggregate + EliminateResult result = eliminateGroupByKeyWithMap(agg, requireOutput); + if (!result.changed) { + return proj; + } + + // Merge into the global replaceMap so that all ancestor nodes get rewritten + replaceMap.putAll(result.replaceMap); + + // Rebuild the child chain with the new aggregate, + // and rewrite the Filter (if present) and Project expressions + Plan newChild; + if (hasFilter) { + Plan updatedFilter = child.withChildren(result.newAgg); + newChild = exprIdReplacer.rewriteExpr(updatedFilter, replaceMap); + } else { + newChild = result.newAgg; + } + Plan newProj = exprIdReplacer.rewriteExpr(proj.withChildren(newChild), replaceMap); + return newProj; + } @Override - public List buildRules() { - return ImmutableList.of( - RuleType.ELIMINATE_GROUP_BY_KEY.build( - logicalProject(logicalAggregate().when(agg -> !agg.getSourceRepeat().isPresent())) - .then(proj -> { - LogicalAggregate agg = proj.child(); - LogicalAggregate newAgg = eliminateGroupByKey(agg, proj.getInputSlots()); - if (newAgg == null) { - return null; - } - return proj.withChildren(newAgg); - })), - RuleType.ELIMINATE_FILTER_GROUP_BY_KEY.build( - logicalProject(logicalFilter(logicalAggregate() - .when(agg -> !agg.getSourceRepeat().isPresent()))) - .then(proj -> { - LogicalAggregate agg = proj.child().child(); - Set requireSlots = new HashSet<>(proj.getInputSlots()); - requireSlots.addAll(proj.child(0).getInputSlots()); - LogicalAggregate newAgg = eliminateGroupByKey(agg, requireSlots); - if (newAgg == null) { - return null; - } - return proj.withChildren(proj.child().withChildren(newAgg)); - }) - ) - ); + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Map replaceMap) { + // When a producer aggregate's output slot is wrapped with any_value(), + // a fresh ExprId is recorded in replaceMap. The CTE consumer's producerToConsumerSlotMap + // still references the old ExprId, so we must rebuild both maps with the new ExprIds. + Map newConsumerToProducer = new LinkedHashMap<>(); + Multimap newProducerToConsumer = LinkedHashMultimap.create(); + for (Slot producerSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { + ExprId newExprId = resolveExprIdChain(producerSlot.getExprId(), replaceMap); + Slot effectiveProducerSlot = newExprId != null + ? (Slot) producerSlot.withExprId(newExprId) + : producerSlot; + for (Slot consumerSlot : cteConsumer.getProducerToConsumerOutputMap().get(producerSlot)) { + newProducerToConsumer.put(effectiveProducerSlot, consumerSlot); + newConsumerToProducer.put(consumerSlot, effectiveProducerSlot); + } + } + return cteConsumer.withTwoMaps(newConsumerToProducer, newProducerToConsumer); + } + + /** Follow transitive ExprId chain to find the final replacement, or null if none. */ + private static ExprId resolveExprIdChain(ExprId exprId, Map replaceMap) { + ExprId newId = replaceMap.get(exprId); + if (newId == null) { + return null; + } + ExprId lastId = newId; + while (true) { + ExprId next = replaceMap.get(lastId); + if (next == null) { + return lastId; + } + lastId = next; + } + } + + /** Result of eliminateGroupByKey: the new aggregate and a map of old->new ExprIds. */ + private static class EliminateResult { + final LogicalAggregate newAgg; + final Map replaceMap; + final boolean changed; + + EliminateResult(LogicalAggregate newAgg, Map replaceMap, boolean changed) { + this.newAgg = newAgg; + this.replaceMap = replaceMap; + this.changed = changed; + } } - LogicalAggregate eliminateGroupByKey(LogicalAggregate agg, Set requireOutput) { - Set removeExpression = findCanBeRemovedExpressions(agg, requireOutput, + EliminateResult eliminateGroupByKeyWithMap(LogicalAggregate agg, Set requireOutput) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, agg.child().getLogicalProperties().getTrait()); + Set removeExpression = result.removeExpression; + Set wrapWithAnyValue = result.wrapWithAnyValue; + List newGroupExpression = new ArrayList<>(); for (Expression expression : agg.getGroupByExpressions()) { - if (!removeExpression.contains(expression)) { + if (!removeExpression.contains(expression) + && !wrapWithAnyValue.contains(expression)) { newGroupExpression.add(expression); } } List newOutput = new ArrayList<>(); + Map replaceMap = new HashMap<>(); + boolean changed = !removeExpression.isEmpty() || !wrapWithAnyValue.isEmpty(); for (NamedExpression expression : agg.getOutputExpressions()) { - if (!removeExpression.contains(expression)) { - newOutput.add(expression); + if (removeExpression.contains(expression)) { + continue; } + if (wrapWithAnyValue.contains(expression)) { + // expression is FD-redundant but needed in output: wrap with any_value + // Use fresh ExprId (auto-generated by Alias) to avoid ExprId collision, + // and record the mapping for rewriting upper plan references. + Alias newAlias = new Alias(new AnyValue(expression.toSlot()), expression.getName()); + replaceMap.put(expression.getExprId(), newAlias.getExprId()); + expression = newAlias; + } + newOutput.add(expression); } - return agg.withGroupByAndOutput(newGroupExpression, newOutput); + return new EliminateResult(agg.withGroupByAndOutput(newGroupExpression, newOutput), replaceMap, changed); } /** - * return removeExpression + * Return expressions that can be completely removed from both group-by and output. + * Kept for backward compatibility with external callers (e.g. PushDownAggThroughJoinOnPkFk). */ public static Set findCanBeRemovedExpressions(LogicalAggregate agg, Set requireOutput, DataTrait dataTrait) { + FindResult result = findCanBeRemovedExpressionsInternal(agg, requireOutput, dataTrait); + return new HashSet<>(result.removeExpression); + } + + /** Result of findCanBeRemovedExpressionsInternal: two sets of expressions. */ + private static class FindResult { + final Set removeExpression; // remove from group-by and output + final Set wrapWithAnyValue; // remove from group-by, wrap with ANY_VALUE in output + + FindResult(Set removeExpression, Set wrapWithAnyValue) { + this.removeExpression = removeExpression; + this.wrapWithAnyValue = wrapWithAnyValue; + } + } + + private static FindResult findCanBeRemovedExpressionsInternal(LogicalAggregate agg, + Set requireOutput, DataTrait dataTrait) { Map> groupBySlots = new HashMap<>(); Set validSlots = new HashSet<>(); for (Expression expression : agg.getGroupByExpressions()) { @@ -110,17 +250,24 @@ public static Set findCanBeRemovedExpressions(LogicalAggregate(); + return new FindResult(new HashSet<>(), new HashSet<>()); } Set> minGroupBySlots = funcDeps.eliminateDeps(new HashSet<>(groupBySlots.values()), requireOutput); Set removeExpression = new HashSet<>(); + Set wrapWithAnyValue = new HashSet<>(); for (Entry> entry : groupBySlots.entrySet()) { - if (!minGroupBySlots.contains(entry.getValue()) - && !requireOutput.containsAll(entry.getValue())) { - removeExpression.add(entry.getKey()); + if (!minGroupBySlots.contains(entry.getValue())) { + // FD redundant: can remove from group-by + if (!requireOutput.containsAll(entry.getValue())) { + // Not needed in output either: remove completely + removeExpression.add(entry.getKey()); + } else { + // Still needed in output: remove from group-by, wrap with ANY_VALUE in output + wrapWithAnyValue.add(entry.getKey()); + } } } - return removeExpression; + return new FindResult(removeExpression, wrapWithAnyValue); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java index c9585d269aa5bd..20ad018fc150f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java @@ -76,7 +76,7 @@ public static Plan rewrite(LogicalAggregate agg, DistinctSelecto // construct cte consumer and aggregate List> newAggs = new ArrayList<>(); // All otherAggFuncs are placed in the first one - Map newToOriginDistinctFuncAlias = new HashMap<>(); + Map newToOriginDistinctFuncAlias = new LinkedHashMap<>(); List outputJoinGroupBys = new ArrayList<>(); for (int i = 0; i < distinctFuncWithAliasReplaced.size(); ++i) { List aliases = distinctFuncWithAliasReplaced.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 96b9981819f97d..ed38140376d729 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -75,7 +75,7 @@ public class CreateMTMVInfo extends CreateTableInfo { public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION," - + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"; + + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, ELIMINATE_GROUP_BY_KEY"; private LogicalPlan logicalQuery; private List simpleColumnDefinitions; private MTMVPartitionDefinition mvPartitionDefinition; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 9385ac2cd5dfd4..ba37e4f77f8197 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -60,7 +60,7 @@ public class PreMaterializedViewRewriterTest extends SqlTestBase { @Test public void testShouldNotRecordTmpPlanWhenNoMv() { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION, ELIMINATE_GROUP_BY_KEY"); BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); new MockUp() { @Mock @@ -2956,6 +2956,19 @@ public void testNeedPreRewrite() { Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext)); } + /** + * Test pre-materialized view rewrite need pre-rewrite when ELIMINATE_GROUP_BY_KEY applied + * */ + @Test + public void testNeedPreRewriteForEliminateGroupByKey() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext("select T1.id from T1"); + StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext(); + statementContext.setForceRecordTmpPlan(true); + statementContext.ruleSetApplied(RuleType.ELIMINATE_GROUP_BY_KEY); + statementContext.getPlannerHooks().add(InitMaterializationContextHook.INSTANCE); + statementContext.getTmpPlanForMvRewrite().add(cascadesContext.getRewritePlan()); + } + private void checkIfEquals(String originalSql, List equivalentSqlList) { // init original cascades context CascadesContext originalCascadesContext = initOriginal(originalSql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 3bc6c736c274e4..8f571904d84f5a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -263,7 +263,7 @@ protected void runBeforeAll() throws Exception { connectContext.getSessionVariable().setDisableNereidsRules( "OLAP_SCAN_PARTITION_PRUNE" + ",PRUNE_EMPTY_PARTITION" - + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + + ",ELIMINATE_GROUP_BY_KEY_BY_UNIFORM" + ",ELIMINATE_GROUP_BY_KEY" + ",ELIMINATE_CONST_JOIN_CONDITION" + ",CONSTANT_PROPAGATION" ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java index 6e6df0909ad2d9..9cc72142e3ea57 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java @@ -135,6 +135,8 @@ void testLeftJoinOnConditionNotRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() + // branch-4.2 keeps all three keys here: the upstream expectation (2 keys) relies on + // uniform-trait derivation for outer-join outputs that master has and branch-4.2 does not. .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3)); } @@ -144,6 +146,8 @@ void testLeftJoinWhereConditionRewrite() { .analyze("select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t2.b,t2.c;") .rewrite() .printlnTree() + // branch-4.2 keeps two keys here: the upstream expectation (only "c") relies on + // uniform-trait derivation for outer-join outputs that master has and branch-4.2 does not. .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 7362c81e5afe0b..a470a1d7a77907 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -18,18 +18,29 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.properties.FuncDeps; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.HashMap; +import java.util.Map; import java.util.Set; class EliminateGroupByKeyTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -97,19 +108,22 @@ void testEliminateTree() { @Test void testEliminateByUniform() { + // Uniform-based elimination is now handled by EliminateGroupByKeyByUniform. + // EliminateGroupByKey only handles FD-based elimination. PlanChecker.from(connectContext) .analyze("select count(name) from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKeyByUniform()) .printlnTree() .matches(logicalAggregate().when(agg -> - agg.getGroupByExpressions().size() == 1 && agg.getGroupByExpressions().get(0).toSql().equals("name"))); + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } @Test void testProjectAlias() { PlanChecker.from(connectContext) .analyze("select id as c from t1 where id = 1 group by name, id") - .rewrite() + .customRewrite(new EliminateGroupByKey()) .printlnTree() .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1)); @@ -181,6 +195,120 @@ void testEliminateByEqual() { && agg.getGroupByExpressions().get(0).toSql().equals("name"))); } + @Test + void testEliminateByPkWithOutputNeeded() throws Exception { + // Regression: when a group-by key (name) is FD-redundant (id -> name) + // but still appears in SELECT, it should be removed from group-by + // and wrapped with ANY_VALUE in the output. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select id, name, count(*) from t1 group by id, name") + .customRewrite(new EliminateGroupByKey()) + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + + @Test + void testEliminateByPkWithOutputNeededProductionPath() throws Exception { + // Production path: same query through .rewrite() (RuleType.ELIMINATE_GROUP_BY_KEY) + // instead of .customRewrite() (RuleType.TEST_REWRITE). + // Use cross join so the aggregate cannot be constant-folded away. + // Use alias on name to force a Project above the Aggregate, which is + // the entry point that EliminateGroupByKey.visitLogicalProject needs. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name as n, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id") + && agg.getOutputExpressions().stream().anyMatch( + e -> e instanceof Alias + && e.child(0) instanceof AnyValue))); + dropConstraint("alter table t1 drop constraint pk2"); + } + + @Test + void testEliminateByPkDisabled() throws Exception { + // Verify that disable_nereids_rules=ELIMINATE_GROUP_BY_KEY prevents the rule + // from eliminating the FD-redundant group-by key. + // Use cross join so the aggregate cannot be constant-folded away. + addConstraint("alter table t1 add constraint pk2 primary key (id)"); + try { + connectContext.getSessionVariable() + .setDisableNereidsRules("PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY"); + PlanChecker.from(connectContext) + .analyze("select t1.id, t1.name, count(*) from t1 as t1" + + " cross join t1 as t2 group by t1.id, t1.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 2)); + } finally { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + dropConstraint("alter table t1 drop constraint pk2"); + } + } + + @Test + void testNestedAggregateUsesRewrittenRequireOutput() { + // Inner aggregate: GROUP BY id, name on unique-key table 'uni'. + // id → name FD (unique key) eliminates name from group-by, + // wrapping it with any_value(name) as a new alias (new ExprId). + // The outer EliminateGroupByKey must rewrite its project through the + // accumulated replaceMap before computing requireOutput; otherwise the + // stale ExprId would incorrectly cause name to be removed from output. + // Bug: proj.getInputSlots() returned old ExprIds → CheckAfterRewrite fails. + PlanChecker.from(connectContext) + .analyze("select t.id, t.name from " + + "(select id, name from uni group by id, name) t " + + "group by t.id, t.name") + .customRewrite(new EliminateGroupByKey()) + .matches( + logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("id"))); + } + + @Test + void testCteConsumerSlotMapUpdatedByReplaceMap() { + // Verify that visitLogicalCTEConsumer correctly updates the slot maps + // when the replaceMap contains an ExprId replacement from the producer. + Slot oldProducerSlot = new SlotReference("old", IntegerType.INSTANCE, false); + Slot consumerSlot = new SlotReference("cons", IntegerType.INSTANCE, false); + + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + new RelationId(1), new CTEId(0), "cte", + ImmutableMap.of(consumerSlot, oldProducerSlot), + ImmutableMultimap.of(oldProducerSlot, consumerSlot)); + + // Simulate replaceMap with ExprId replacement from aggregate rewrite + ExprId newProducerExprId = new ExprId(999); // fresh Id from any_value alias + Map replaceMap = new HashMap<>(); + replaceMap.put(oldProducerSlot.getExprId(), newProducerExprId); + + EliminateGroupByKey rewriter = new EliminateGroupByKey(); + LogicalCTEConsumer updated = (LogicalCTEConsumer) rewriter.visitLogicalCTEConsumer( + consumer, replaceMap); + + // The updated consumer's producerToConsumerSlotMap should be keyed by the new ExprId + Multimap updatedMap = updated.getProducerToConsumerOutputMap(); + Assertions.assertEquals(1, updatedMap.keySet().size()); + Slot updatedProducerKey = updatedMap.keySet().iterator().next(); + Assertions.assertEquals(newProducerExprId, updatedProducerKey.getExprId(), + "Producer slot ExprId should be updated to the new one from replaceMap"); + Assertions.assertTrue(updatedMap.get(updatedProducerKey).contains(consumerSlot), + "Consumer slot should still be mapped"); + } + @Test void testRepeatEliminateByEqual() { PlanChecker.from(connectContext) diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy index 0fa49496708967..e67c5f976d271b 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy @@ -84,7 +84,7 @@ suite("eliminate_gby_key") { select t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -144,7 +144,7 @@ suite("eliminate_gby_key") { select t2_c2, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -184,7 +184,7 @@ suite("eliminate_gby_key") { select c3, t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -264,7 +264,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[c1#13, c3#18]") } explain { @@ -284,7 +284,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1, cnt from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18,") + contains("groupByExpr=[c1#13, c3#18]") } sql "drop table if exists eli_gbk_t" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy index d12442a26f1716..458e8c78b06509 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy @@ -22,7 +22,7 @@ suite("agg_variety") { sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 1b936f5a609a59..6032899049a0de 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -25,7 +25,7 @@ suite("aggregate_without_roll_up") { sql "SET enable_dphyp_optimizer = false;" sql "SET max_table_count_use_cascades_join_reorder = 20;" sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders """ @@ -1688,7 +1688,7 @@ suite("aggregate_without_roll_up") { order_qt_query29_0_before "${query29_0}" async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0") order_qt_query29_0_after "${query29_0}" - sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + // sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" // query and mv has the same filter but position is different, should rewrite successfully @@ -1839,6 +1839,7 @@ suite("aggregate_without_roll_up") { 13, 14; """ + order_qt_query30_0_before "${query30_0}" async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0", [TRY_IN_RBO, FORCE_IN_RBO]) // ELIMINATE_CONST_JOIN_CONDITION not work, so should success @@ -1846,7 +1847,6 @@ suite("aggregate_without_roll_up") { order_qt_query30_0_after "${query30_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0""" - // query and mv has the same filter but position is different, should rewrite successfully // query join condition has alias def mv31_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy index a2fcb2eba15913..24cc3d234b4ba8 100644 --- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy @@ -23,6 +23,8 @@ suite("mtmv_range_date_datetrunc_date_part_up") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" + String mv_prefix = "range_datetrunc_date_up" String tb_name = mv_prefix + "_tb" String mv_name = mv_prefix + "_mv" diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy index 58082d74decf05..66231cff69f901 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/with_lock/dml_rewrite_with_lock.groovy @@ -22,139 +22,140 @@ suite("dml_rewrite_with_lock", "zfr_mtmv_test") { sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite=true" sql "SET enable_materialized_view_union_rewrite=true" - - sql """ - drop table if exists lineitem_range_date_union - """ - - sql """CREATE TABLE `lineitem_range_date_union` ( - `l_orderkey` BIGINT NULL, - `l_linenumber` INT NULL, - `l_partkey` INT NULL, - `l_suppkey` INT NULL, - `l_quantity` DECIMAL(15, 2) NULL, - `l_extendedprice` DECIMAL(15, 2) NULL, - `l_discount` DECIMAL(15, 2) NULL, - `l_tax` DECIMAL(15, 2) NULL, - `l_returnflag` VARCHAR(1) NULL, - `l_linestatus` VARCHAR(1) NULL, - `l_commitdate` DATE NULL, - `l_receiptdate` DATE NULL, - `l_shipinstruct` VARCHAR(25) NULL, - `l_shipmode` VARCHAR(10) NULL, - `l_comment` VARCHAR(44) NULL, - `l_shipdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) - COMMENT 'OLAP' - partition by range (`l_shipdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01"))) - DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - drop table if exists orders_range_date_union - """ - - sql """CREATE TABLE `orders_range_date_union` ( - `o_orderkey` BIGINT NULL, - `o_custkey` INT NULL, - `o_orderstatus` VARCHAR(1) NULL, - `o_totalprice` DECIMAL(15, 2) NULL, - `o_orderpriority` VARCHAR(15) NULL, - `o_clerk` VARCHAR(15) NULL, - `o_shippriority` INT NULL, - `o_comment` VARCHAR(79) NULL, - `o_orderdate` DATE not NULL - ) ENGINE=OLAP - DUPLICATE KEY(`o_orderkey`, `o_custkey`) - COMMENT 'OLAP' - partition by range (`o_orderdate`) ( - partition p1 values [("2023-10-29"), ("2023-10-30")), - partition p2 values [("2023-10-30"), ("2023-10-31")), - partition p3 values [("2023-10-31"), ("2023-11-01")), - partition p4 values [("2023-11-01"), ("2023-11-02")), - partition p5 values [("2023-11-02"), ("2023-11-03"))) - DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - );""" - - sql """ - insert into lineitem_range_date_union values - (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), - (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), - (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), - (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); - """ - - sql """ - insert into orders_range_date_union values - (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), - (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), - (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), - (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), - (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), - (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), - (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), - (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), - (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); - """ - - sql """DROP MATERIALIZED VIEW if exists day_mv;""" - create_async_mv(db, "day_mv", - """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """ - ) - - def query1 = """ - select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey - from lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - mv_rewrite_success(query1, "day_mv") - - def query2 = """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey - group by col1, l_shipdate, l_orderkey - """ - - sql """DROP MATERIALIZED VIEW if exists hour_mv;""" - create_async_mv(db, "hour_mv", - """ - select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from - lineitem_range_date_union as t1 left join orders_range_date_union as t2 - on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; - """) - mv_rewrite_success(query2, "hour_mv") - - - sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" - sql """insert into lineitem_range_date_union values - (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" - - sql """refresh MATERIALIZED VIEW hour_mv auto;""" - waitingMTMVTaskFinishedByMvName("hour_mv") - - sql """refresh MATERIALIZED VIEW day_mv auto;""" - waitingMTMVTaskFinishedByMvName("day_mv") - - mv_rewrite_success(query1, "day_mv") - mv_rewrite_success(query2, "hour_mv") + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" + + // sql """ + // drop table if exists lineitem_range_date_union + // """ + + // sql """CREATE TABLE `lineitem_range_date_union` ( + // `l_orderkey` BIGINT NULL, + // `l_linenumber` INT NULL, + // `l_partkey` INT NULL, + // `l_suppkey` INT NULL, + // `l_quantity` DECIMAL(15, 2) NULL, + // `l_extendedprice` DECIMAL(15, 2) NULL, + // `l_discount` DECIMAL(15, 2) NULL, + // `l_tax` DECIMAL(15, 2) NULL, + // `l_returnflag` VARCHAR(1) NULL, + // `l_linestatus` VARCHAR(1) NULL, + // `l_commitdate` DATE NULL, + // `l_receiptdate` DATE NULL, + // `l_shipinstruct` VARCHAR(25) NULL, + // `l_shipmode` VARCHAR(10) NULL, + // `l_comment` VARCHAR(44) NULL, + // `l_shipdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) + // COMMENT 'OLAP' + // partition by range (`l_shipdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01"))) + // DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // drop table if exists orders_range_date_union + // """ + + // sql """CREATE TABLE `orders_range_date_union` ( + // `o_orderkey` BIGINT NULL, + // `o_custkey` INT NULL, + // `o_orderstatus` VARCHAR(1) NULL, + // `o_totalprice` DECIMAL(15, 2) NULL, + // `o_orderpriority` VARCHAR(15) NULL, + // `o_clerk` VARCHAR(15) NULL, + // `o_shippriority` INT NULL, + // `o_comment` VARCHAR(79) NULL, + // `o_orderdate` DATE not NULL + // ) ENGINE=OLAP + // DUPLICATE KEY(`o_orderkey`, `o_custkey`) + // COMMENT 'OLAP' + // partition by range (`o_orderdate`) ( + // partition p1 values [("2023-10-29"), ("2023-10-30")), + // partition p2 values [("2023-10-30"), ("2023-10-31")), + // partition p3 values [("2023-10-31"), ("2023-11-01")), + // partition p4 values [("2023-11-01"), ("2023-11-02")), + // partition p5 values [("2023-11-02"), ("2023-11-03"))) + // DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1" + // );""" + + // sql """ + // insert into lineitem_range_date_union values + // (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'), + // (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'), + // (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'), + // (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'); + // """ + + // sql """ + // insert into orders_range_date_union values + // (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'), + // (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'), + // (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'), + // (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'), + // (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'), + // (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'), + // (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'), + // (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'), + // (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); + // """ + + // sql """DROP MATERIALIZED VIEW if exists day_mv;""" + // create_async_mv(db, "day_mv", + // """select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """ + // ) + + // def query1 = """ + // select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey + // from lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // mv_rewrite_success(query1, "day_mv") + + // def query2 = """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey + // group by col1, l_shipdate, l_orderkey + // """ + + // sql """DROP MATERIALIZED VIEW if exists hour_mv;""" + // create_async_mv(db, "hour_mv", + // """ + // select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from + // lineitem_range_date_union as t1 left join orders_range_date_union as t2 + // on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey; + // """) + // mv_rewrite_success(query2, "hour_mv") + + + // sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));""" + // sql """insert into lineitem_range_date_union values + // (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')""" + + // sql """refresh MATERIALIZED VIEW hour_mv auto;""" + // waitingMTMVTaskFinishedByMvName("hour_mv") + + // sql """refresh MATERIALIZED VIEW day_mv auto;""" + // waitingMTMVTaskFinishedByMvName("day_mv") + + // mv_rewrite_success(query1, "day_mv") + // mv_rewrite_success(query2, "hour_mv") } diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy index 789959e4ccccfd..724a5cc15c9aa9 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_list_str_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_list_str_increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_1 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy index 36288e1d7f7fb6..e32bb5e2742996 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_date_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_date_increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_2 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy index 89be330e2b0ad1..f68e25fee360a5 100644 --- a/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/increment_create/inner_join_range_number_increment_create.groovy @@ -21,7 +21,7 @@ suite("inner_join_range_number_increment_create") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=false" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_inner_3 """ diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 89e927c3f93a77..b31e3ca53dafea 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -22,7 +22,7 @@ suite("nested_mtmv") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" - + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY_KEY'" sql """ drop table if exists orders_1 """