diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index af7b21e259491c..3fe0ba94c8e47b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -65,7 +65,6 @@ import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.VariableMgr; import org.apache.doris.resource.Tag; -import org.apache.doris.resource.computegroup.ComputeGroup; import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.statistics.AnalysisInfo; @@ -75,16 +74,12 @@ import org.apache.doris.statistics.OlapAnalysisTask; import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.system.Backend; -import org.apache.doris.system.BeSelectionPolicy; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TEncryptionAlgorithm; -import org.apache.doris.thrift.TFetchOption; import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; -import org.apache.doris.thrift.TNodeInfo; import org.apache.doris.thrift.TOlapTable; -import org.apache.doris.thrift.TPaloNodesInfo; import org.apache.doris.thrift.TPatternType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TSortType; @@ -106,7 +101,6 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; import lombok.Setter; -import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -3174,48 +3168,6 @@ public AutoIncrementGenerator getAutoIncrementGenerator() { return autoIncrementGenerator; } - /** - * generate two phase read fetch option from this olap table. - * - * @param selectedIndexId the index want to scan - */ - public TFetchOption generateTwoPhaseReadOption(long selectedIndexId) { - boolean useStoreRow = this.storeRowColumn() - && CollectionUtils.isEmpty(getTableProperty().getCopiedRowStoreColumns()); - TFetchOption fetchOption = new TFetchOption(); - fetchOption.setFetchRowStore(useStoreRow); - fetchOption.setUseTwoPhaseFetch(true); - - ConnectContext context = ConnectContext.get(); - if (context == null) { - context = new ConnectContext(); - } - BeSelectionPolicy policy = new BeSelectionPolicy.Builder() - .needQueryAvailable() - .setRequireAliveBe() - .build(); - - TPaloNodesInfo nodesInfo = new TPaloNodesInfo(); - ComputeGroup computeGroup = context.getComputeGroupSafely(); - - if (ComputeGroup.INVALID_COMPUTE_GROUP.equals(computeGroup)) { - throw new RuntimeException(ComputeGroup.INVALID_COMPUTE_GROUP_ERR_MSG); - } - - for (Backend backend : policy.getCandidateBackends(computeGroup.getBackendList())) { - nodesInfo.addToNodes(new TNodeInfo(backend.getId(), 0, backend.getHost(), backend.getBrpcPort())); - } - - fetchOption.setNodesInfo(nodesInfo); - - if (!useStoreRow) { - List columnsDesc = Lists.newArrayList(); - getColumnDesc(selectedIndexId, columnsDesc, null, null); - fetchOption.setColumnDesc(columnsDesc); - } - return fetchOption; - } - public void getColumnDesc(long selectedIndexId, List columnsDesc, List keyColumnNames, List keyColumnTypes, Set materializedColumnNames) { if (selectedIndexId != -1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java index e942e71d064e64..1e6393f162c774 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java @@ -41,8 +41,6 @@ import org.apache.doris.nereids.trees.plans.PlanNodeAndHash; import org.apache.doris.nereids.trees.plans.algebra.OlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; @@ -207,12 +205,6 @@ public Cost visitPhysicalFilter(PhysicalFilter filter, PlanConte (filter.getConjuncts().size() - prefixIndexMatched + exprCost) * filterCostFactor); } - @Override - public Cost visitPhysicalDeferMaterializeOlapScan(PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, - PlanContext context) { - return visitPhysicalOlapScan(deferMaterializeOlapScan.getPhysicalOlapScan(), context); - } - public Cost visitPhysicalSchemaScan(PhysicalSchemaScan physicalSchemaScan, PlanContext context) { Statistics statistics = context.getStatisticsWithCheck(); return Cost.ofCpu(context.getSessionVariable(), statistics.getRowCount()); @@ -297,12 +289,6 @@ public Cost visitPhysicalTopN(PhysicalTopN topN, PlanContext con return Cost.of(context.getSessionVariable(), childRowCount, rowCount, childRowCount); } - @Override - public Cost visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN topN, - PlanContext context) { - return visitPhysicalTopN(topN.getPhysicalTopN(), context); - } - @Override public Cost visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, PlanContext context) { Statistics statistics = context.getStatisticsWithCheck(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index be66a89526c57d..842996fe2974d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -133,9 +133,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalDictionarySink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; @@ -248,7 +245,6 @@ import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; -import org.apache.doris.thrift.TFetchOption; import org.apache.doris.thrift.TPaimonFixedBucketInfo; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TPushAggOp; @@ -508,16 +504,6 @@ public PlanFragment visitPhysicalResultSink(PhysicalResultSink p return planFragment; } - @Override - public PlanFragment visitPhysicalDeferMaterializeResultSink( - PhysicalDeferMaterializeResultSink sink, - PlanTranslatorContext context) { - PlanFragment planFragment = visitPhysicalResultSink(sink.getPhysicalResultSink(), context); - TFetchOption fetchOption = sink.getOlapTable().generateTwoPhaseReadOption(sink.getSelectedIndexId()); - ((ResultSink) planFragment.getSink()).setFetchOption(fetchOption); - return planFragment; - } - @Override public PlanFragment visitPhysicalDictionarySink(PhysicalDictionarySink dictionarySink, PlanTranslatorContext context) { @@ -1146,17 +1132,6 @@ private void translateRuntimeFilter(PhysicalRelation physicalRelation, ScanNode context.getTopnFilterContext().translateTarget(physicalRelation, scanNode, context); } - @Override - public PlanFragment visitPhysicalDeferMaterializeOlapScan( - PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, PlanTranslatorContext context) { - PlanFragment planFragment = visitPhysicalOlapScan(deferMaterializeOlapScan.getPhysicalOlapScan(), context); - OlapScanNode olapScanNode = (OlapScanNode) planFragment.getPlanRoot(); - TupleDescriptor tupleDescriptor = context.getTupleDesc(olapScanNode.getTupleId()); - context.createSlotDesc(tupleDescriptor, deferMaterializeOlapScan.getColumnIdSlot()); - context.getTopnFilterContext().translateTarget(deferMaterializeOlapScan, olapScanNode, context); - return planFragment; - } - @Override public PlanFragment visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, PlanTranslatorContext context) { @@ -2316,12 +2291,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject project List slots = null; // TODO FE/BE do not support multi-layer-project on MultiDataSink now. if (project.hasMultiLayerProjection() - && !(inputFragment instanceof MultiCastPlanFragment) - // TODO support for two phase read with project, remove it after refactor - && !(project.child() instanceof PhysicalDeferMaterializeTopN) - && !(project.child() instanceof PhysicalDeferMaterializeOlapScan - || (project.child() instanceof PhysicalFilter - && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { + && !(inputFragment instanceof MultiCastPlanFragment)) { int layerCount = project.getMultiLayerProjects().size(); for (int i = 0; i < layerCount; i++) { List layer = project.getMultiLayerProjects().get(i); @@ -2438,28 +2408,20 @@ public PlanFragment visitPhysicalProject(PhysicalProject project } if (inputPlanNode instanceof ScanNode) { - // TODO support for two phase read with project, remove this if after refactor - if (!(project.child() instanceof PhysicalDeferMaterializeOlapScan - || (project.child() instanceof PhysicalFilter - && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { - TupleDescriptor projectionTuple = generateTupleDesc(slots, - ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(projectionTuple); - } + TupleDescriptor projectionTuple = generateTupleDesc(slots, + ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(projectionTuple); + if (inputPlanNode instanceof OlapScanNode) { ((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet); } updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet, requiredByProjectSlotIdSet, context); } else { - if (project.child() instanceof PhysicalDeferMaterializeTopN) { - inputFragment.setOutputExprs(allProjectionExprs); - } else { - TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(tupleDescriptor); - } + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(tupleDescriptor); } return inputFragment; } @@ -2750,21 +2712,6 @@ public PlanFragment visitPhysicalTopN(PhysicalTopN topN, PlanTra return inputFragment; } - @Override - public PlanFragment visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN topN, - PlanTranslatorContext context) { - PlanFragment planFragment = visitPhysicalTopN(topN.getPhysicalTopN(), context); - if (planFragment.getPlanRoot() instanceof SortNode) { - SortNode sortNode = (SortNode) planFragment.getPlanRoot(); - sortNode.setUseTwoPhaseReadOpt(true); - sortNode.getSortInfo().setUseTwoPhaseRead(); - if (context.getTopnFilterContext().isTopnFilterSource(topN)) { - context.getTopnFilterContext().translateSource(topN, sortNode); - } - } - return planFragment; - } - @Override public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, PlanTranslatorContext context) { PlanFragment inputPlanFragment = repeat.child(0).accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 9167a0aed681dd..833967447b16a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -63,7 +63,6 @@ import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; import org.apache.doris.nereids.rules.rewrite.DecomposeRepeatWithPreAggregation; import org.apache.doris.nereids.rules.rewrite.DecoupleEncodeDecode; -import org.apache.doris.nereids.rules.rewrite.DeferMaterializeTopNResult; import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector; import org.apache.doris.nereids.rules.rewrite.DistinctAggregateRewriter; import org.apache.doris.nereids.rules.rewrite.DistinctWindowExpression; @@ -797,9 +796,6 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(new PushDownScoreTopNIntoOlapScan(), new CheckScoreUsage()) ), - topic("topn optimize", - topDown(new DeferMaterializeTopNResult()) - ), topic("add projection for join", custom(RuleType.ADD_PROJECT_FOR_JOIN, AddProjectForJoin::new), topDown(new MergeProjectable()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/lineage/LineageInfoExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/lineage/LineageInfoExtractor.java index fefdb0a42d3089..10e6029a4ea77c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/lineage/LineageInfoExtractor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/lineage/LineageInfoExtractor.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; @@ -352,21 +351,6 @@ public Void visitLogicalTopN(LogicalTopN topN, LineageInfo linea return super.visitLogicalTopN(topN, lineageInfo); } - /** - * Collect SORT indirect lineage from defer-materialize TopN. - * - *

Using the example SQL above, there is no defer-materialize TopN. - */ - @Override - public Void visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, - LineageInfo lineageInfo) { - List sortExprs = new ArrayList<>(); - topN.getOrderKeys().forEach(orderKey -> sortExprs.add(orderKey.getExpr())); - Set shuttled = shuttleExpressions(sortExprs, exprIdExpressionMap); - addIndirectLineage(IndirectLineageType.SORT, shuttled, lineageInfo); - return super.visitLogicalDeferMaterializeTopN(topN, lineageInfo); - } - /** * Replace UNION outputs in direct lineage using child outputs. * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java index 40c522232f84bc..58134ec5d91f9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java @@ -34,7 +34,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalBlackholeSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDictionarySink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -303,13 +302,6 @@ public Plan visitPhysicalDictionarySink(PhysicalDictionarySink d return rewriteUnary(dictionarySink, ctx.withAllowShuffleKeyPrune(childAllowShuffleKeyPrune)); } - @Override - public Plan visitPhysicalDeferMaterializeResultSink( - PhysicalDeferMaterializeResultSink sink, - PruneCtx ctx) { - return rewriteUnary(sink, ctx.withAllowShuffleKeyPrune(false)); - } - private

> P rewriteUnary(P plan, PruneCtx ctx) { Plan oldChild = plan.child(); Plan newChild = oldChild.accept(this, ctx); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java index 377d47ca6d4b27..9e6807cc0b2905 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.SortPhase; import org.apache.doris.nereids.trees.plans.algebra.TopN; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; /** @@ -50,17 +49,11 @@ public PhysicalTopN visitPhysicalTopN(PhysicalTopN topN, - CascadesContext ctx) { - topN.child().accept(this, ctx); - if (checkTopN(topN)) { - TopnFilterPushDownVisitor pusher = new TopnFilterPushDownVisitor(ctx.getTopnFilterContext()); - TopnFilterPushDownVisitor.PushDownContext pushdownContext = new PushDownContext(topN, - topN.getOrderKeys().get(0).getExpr(), - topN.getOrderKeys().get(0).isNullFirst()); - topN.accept(pusher, pushdownContext); - } - return topN; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java index 789ab2c0759c5e..a36d807316895f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; @@ -269,7 +268,6 @@ private boolean supportPhysicalRelations(PhysicalRelation relation) { || relation instanceof PhysicalEsScan || relation instanceof PhysicalFileScan || relation instanceof PhysicalJdbcScan - || relation instanceof PhysicalDeferMaterializeOlapScan || relation instanceof PhysicalLazyMaterializeOlapScan; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 8c89d45891bdb9..c2d289ed8b67f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -35,7 +35,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; @@ -184,12 +183,6 @@ public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanC return new PhysicalProperties(olapScan.getDistributionSpec()); } - @Override - public PhysicalProperties visitPhysicalDeferMaterializeOlapScan( - PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, PlanContext context) { - return visitPhysicalOlapScan(deferMaterializeOlapScan.getPhysicalOlapScan(), context); - } - @Override public PhysicalProperties visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, PlanContext context) { return PhysicalProperties.GATHER; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index e95eabceec692a..84a96996cd071b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalBlackholeSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDictionarySink; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -252,14 +251,6 @@ public Void visitPhysicalResultSink(PhysicalResultSink physicalR return null; } - @Override - public Void visitPhysicalDeferMaterializeResultSink( - PhysicalDeferMaterializeResultSink sink, - PlanContext context) { - addRequestPropertyToChildren(PhysicalProperties.GATHER); - return null; - } - /* ******************************************************************************************** * Other Node, in lexicographical order * ******************************************************************************************** */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 908ce4365418fe..32269acac4365e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -63,9 +63,6 @@ import org.apache.doris.nereids.rules.implementation.LogicalCTEAnchorToPhysicalCTEAnchor; import org.apache.doris.nereids.rules.implementation.LogicalCTEConsumerToPhysicalCTEConsumer; import org.apache.doris.nereids.rules.implementation.LogicalCTEProducerToPhysicalCTEProducer; -import org.apache.doris.nereids.rules.implementation.LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.rules.implementation.LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink; -import org.apache.doris.nereids.rules.implementation.LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN; import org.apache.doris.nereids.rules.implementation.LogicalDictionarySinkToPhysicalDictionarySink; import org.apache.doris.nereids.rules.implementation.LogicalEmptyRelationToPhysicalEmptyRelation; import org.apache.doris.nereids.rules.implementation.LogicalEsScanToPhysicalEsScan; @@ -209,7 +206,6 @@ public class RuleSet { .add(new LogicalJoinToHashJoin()) .add(new LogicalJoinToNestedLoopJoin()) .add(new LogicalOlapScanToPhysicalOlapScan()) - .add(new LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan()) .add(new LogicalSchemaScanToPhysicalSchemaScan()) .add(new LogicalHudiScanToPhysicalHudiScan()) .add(new LogicalFileScanToPhysicalFileScan()) @@ -222,7 +218,6 @@ public class RuleSet { .add(new LogicalWindowToPhysicalWindow()) .add(new LogicalSortToPhysicalQuickSort()) .add(new LogicalTopNToPhysicalTopN()) - .add(new LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN()) .add(new LogicalPartitionTopNToPhysicalPartitionTopN()) .add(new LogicalAssertNumRowsToPhysicalAssertNumRows()) .add(new LogicalOneRowRelationToPhysicalOneRowRelation()) @@ -249,7 +244,6 @@ public class RuleSet { .add(new LogicalJdbcTableSinkToPhysicalJdbcTableSink()) .add(new LogicalFileSinkToPhysicalFileSink()) .add(new LogicalResultSinkToPhysicalResultSink()) - .add(new LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink()) .add(new LogicalDictionarySinkToPhysicalDictionarySink()) .add(new LogicalBlackholeSinkToPhysicalBlackholeSink()) .add(new LogicalTVFTableSinkToPhysicalTVFTableSink()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 25f46eba8e563c..cf077b37d25730 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -421,8 +421,6 @@ public enum RuleType { OPERATIVE_COLUMN_DERIVE(RuleTypeClass.REWRITE), - // topn opts - DEFER_MATERIALIZE_TOP_N_RESULT(RuleTypeClass.REWRITE), // short circuit rule SHOR_CIRCUIT_POINT_QUERY(RuleTypeClass.REWRITE), // skew rewrtie @@ -547,12 +545,10 @@ public enum RuleType { LOGICAL_REPEAT_TO_PHYSICAL_REPEAT_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_SORT_TO_PHYSICAL_QUICK_SORT_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_TOP_N_TO_PHYSICAL_TOP_N_RULE(RuleTypeClass.IMPLEMENTATION), - LOGICAL_DEFER_MATERIALIZE_TOP_N_TO_PHYSICAL_DEFER_MATERIALIZE_TOP_N_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_PARTITION_TOP_N_TO_PHYSICAL_PARTITION_TOP_N_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_EMPTY_RELATION_TO_PHYSICAL_EMPTY_RELATION_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_LIMIT_TO_PHYSICAL_LIMIT_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), - LOGICAL_DEFER_MATERIALIZE_OLAP_SCAN_TO_PHYSICAL_DEFER_MATERIALIZE_OLAP_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_SCHEMA_SCAN_TO_PHYSICAL_SCHEMA_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_HUDI_SCAN_TO_PHYSICAL_HUDI_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), @@ -570,7 +566,6 @@ public enum RuleType { LOGICAL_ICEBERG_MERGE_SINK_TO_PHYSICAL_ICEBERG_MERGE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_JDBC_TABLE_SINK_TO_PHYSICAL_JDBC_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION), - LOGICAL_DEFER_MATERIALIZE_RESULT_SINK_TO_PHYSICAL_DEFER_MATERIALIZE_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_FILE_SINK_TO_PHYSICAL_FILE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_DICTIONARY_SINK_TO_PHYSICAL_DICTIONARY_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_TVF_TABLE_SINK_TO_PHYSICAL_TVF_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 86b42885387f7e..9c2cc60c6a019c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.algebra.Generate; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -215,8 +214,7 @@ private boolean isLegacyVariant(DataType dataType) { private void checkMatchIsUsedCorrectly(Plan plan) { for (Expression expression : plan.getExpressions()) { if (expression instanceof Match) { - if (plan instanceof LogicalFilter && (plan.child(0) instanceof LogicalOlapScan - || plan.child(0) instanceof LogicalDeferMaterializeOlapScan)) { + if (plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan) { return; } else { throw new AnalysisException(String.format( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index d8d903384ba325..2a7c75372998e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -115,7 +115,6 @@ public List buildRules() { new LogicalJdbcTableSinkRewrite().build(), new LogicalOlapTableSinkRewrite().build(), new LogicalDictionarySinkRewrite().build(), - new LogicalDeferMaterializeResultSinkRewrite().build(), new LogicalOlapTableSinkExpressionRewrite().build()); } @@ -560,14 +559,6 @@ public Rule build() { } } - private class LogicalDeferMaterializeResultSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalDeferMaterializeResultSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - private LogicalSink applyRewriteToSink(MatchingContext> ctx) { LogicalSink sink = ctx.root; ExpressionRewriteContext context = new ExpressionRewriteContext(sink, ctx.cascadesContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan.java deleted file mode 100644 index 8d30f32dbc6dcc..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.implementation; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; - -import java.util.Optional; - -/** - * implement defer materialize olap scan from logical to physical - */ -public class LogicalDeferMaterializeOlapScanToPhysicalDeferMaterializeOlapScan extends OneImplementationRuleFactory { - @Override - public Rule build() { - return logicalDeferMaterializeOlapScan().thenApply(ctx -> { - LogicalDeferMaterializeOlapScan logicalDeferOlapScan = ctx.root; - PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan().build() - .transform(logicalDeferOlapScan.getLogicalOlapScan(), ctx.cascadesContext).get(0); - return new PhysicalDeferMaterializeOlapScan(physicalOlapScan, - logicalDeferOlapScan.getDeferMaterializeSlotIds(), - logicalDeferOlapScan.getColumnIdSlot(), - Optional.empty(), - logicalDeferOlapScan.getLogicalProperties()); - }).toRule(RuleType.LOGICAL_DEFER_MATERIALIZE_OLAP_SCAN_TO_PHYSICAL_DEFER_MATERIALIZE_OLAP_SCAN_RULE); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink.java deleted file mode 100644 index 0c02841e2b8384..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink.java +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.implementation; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeResultSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; - -import java.util.Optional; - -/** - * implement defer materialize result sink from logical to physical - */ -public class LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink - extends OneImplementationRuleFactory { - @Override - public Rule build() { - return logicalDeferMaterializeResultSink().thenApply(ctx -> { - LogicalDeferMaterializeResultSink sink = ctx.root; - PhysicalResultSink physicalResultSink - = (PhysicalResultSink) new LogicalResultSinkToPhysicalResultSink() - .build() - .transform(sink.getLogicalResultSink(), ctx.cascadesContext) - .get(0); - return new PhysicalDeferMaterializeResultSink<>( - physicalResultSink, sink.getOlapTable(), sink.getSelectedIndexId(), - Optional.empty(), sink.getLogicalProperties(), sink.child()); - }).toRule(RuleType.LOGICAL_DEFER_MATERIALIZE_RESULT_SINK_TO_PHYSICAL_DEFER_MATERIALIZE_RESULT_SINK_RULE); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN.java deleted file mode 100644 index 2799ca30147bc7..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN.java +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.implementation; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.SortPhase; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; -import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; - -import java.util.Optional; - -/** - * implement defer materialize top n from logical to physical - */ -public class LogicalDeferMaterializeTopNToPhysicalDeferMaterializeTopN extends OneImplementationRuleFactory { - @Override - public Rule build() { - return logicalDeferMaterializeTopN().thenApply(ctx -> { - LogicalDeferMaterializeTopN topN = ctx.root; - PhysicalTopN physicalTopN = (PhysicalTopN) new LogicalTopNToPhysicalTopN() - .build() - .transform(topN.getLogicalTopN(), ctx.cascadesContext) - .get(0); - if (physicalTopN.getSortPhase() == SortPhase.MERGE_SORT) { - return wrap(physicalTopN, topN, wrap((PhysicalTopN) physicalTopN.child(), topN, - ((PhysicalTopN) physicalTopN.child()).child())); - } else { - return wrap(physicalTopN, topN, physicalTopN.child()); - } - - }).toRule(RuleType.LOGICAL_DEFER_MATERIALIZE_TOP_N_TO_PHYSICAL_DEFER_MATERIALIZE_TOP_N_RULE); - } - - private PhysicalDeferMaterializeTopN wrap(PhysicalTopN physicalTopN, - LogicalDeferMaterializeTopN logicalWrapped, Plan child) { - return new PhysicalDeferMaterializeTopN<>(physicalTopN, - logicalWrapped.getDeferMaterializeSlotIds(), logicalWrapped.getColumnIdSlot(), - Optional.empty(), logicalWrapped.getLogicalProperties(), child); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java deleted file mode 100644 index 48b5ad4c931176..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ /dev/null @@ -1,337 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Type; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeResultSink; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; -import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * rewrite simple top n query to defer materialize slot not use for sort or predicate - */ -public class DeferMaterializeTopNResult implements RewriteRuleFactory { - - @Override - public List buildRules() { - return ImmutableList.of( - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink( - logicalTopN( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> t.getOrderKeys().stream() - .map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - ).then(r -> deferMaterialize(r, r.child(), - Optional.empty(), Optional.empty(), r.child().child())) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink( - logicalTopN( - logicalProject( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - ) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> { - for (OrderKey orderKey : t.getOrderKeys()) { - if (!orderKey.getExpr().isColumnFromTable()) { - return false; - } - if (!(orderKey.getExpr() instanceof SlotReference)) { - return false; - } - SlotReference slotRef = (SlotReference) orderKey.getExpr(); - // do not support alias in project now - if (!t.child().getProjects().contains(slotRef)) { - return false; - } - } - return true; - }) - ).then(r -> { - LogicalProject project = r.child().child(); - return deferMaterialize(r, r.child(), Optional.of(project), - Optional.empty(), project.child()); - }) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink( - logicalTopN( - logicalFilter( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - ) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> t.getOrderKeys().stream() - .map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - ).then(r -> { - LogicalFilter filter = r.child().child(); - return deferMaterialize(r, r.child(), Optional.empty(), - Optional.of(filter), filter.child()); - }) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink( - logicalTopN( - logicalProject( - logicalFilter( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - ) - ) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> { - for (OrderKey orderKey : t.getOrderKeys()) { - if (!orderKey.getExpr().isColumnFromTable()) { - return false; - } - if (!(orderKey.getExpr() instanceof SlotReference)) { - return false; - } - SlotReference slotRef = (SlotReference) orderKey.getExpr(); - // do not support alias in project now - if (!t.child().getProjects().contains(slotRef)) { - return false; - } - } - return true; - }) - ).then(r -> { - LogicalProject> project = r.child().child(); - LogicalFilter filter = project.child(); - return deferMaterialize(r, r.child(), Optional.of(project), - Optional.of(filter), filter.child()); - }) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalProject( - logicalTopN( - logicalProject( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - - ) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> { - for (OrderKey orderKey : t.getOrderKeys()) { - if (!orderKey.getExpr().isColumnFromTable()) { - return false; - } - if (!(orderKey.getExpr() instanceof SlotReference)) { - return false; - } - SlotReference slotRef = (SlotReference) orderKey.getExpr(); - // do not support alias in project now - if (!t.child().getProjects().contains(slotRef)) { - return false; - } - } - return true; - }) - ).when(project -> project.canMergeChildProjections(project.child().child()))).then(r -> { - LogicalProject upperProject = r.child(); - LogicalProject bottomProject = r.child().child().child(); - Optional> projections = upperProject.mergeProjections(bottomProject); - if (!projections.isPresent()) { - return null; - } - LogicalProject project = upperProject.withProjects(projections.get()); - return deferMaterialize(r, r.child().child(), Optional.of(project), - Optional.empty(), bottomProject.child()); - }) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalProject( - logicalTopN( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> { - for (OrderKey orderKey : t.getOrderKeys()) { - if (!orderKey.getExpr().isColumnFromTable()) { - return false; - } - if (!(orderKey.getExpr() instanceof SlotReference)) { - return false; - } - } - return true; - }) - )).then(r -> deferMaterialize(r, r.child().child(), Optional.of(r.child()), - Optional.empty(), r.child().child().child())) - ), - RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalProject( - logicalTopN( - logicalProject(logicalFilter( - logicalOlapScan() - .when(s -> s.getTable().getEnableLightSchemaChange()) - .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) - ) - ) - ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) - .whenNot(t -> t.getOrderKeys().isEmpty()) - .when(t -> { - for (OrderKey orderKey : t.getOrderKeys()) { - if (!orderKey.getExpr().isColumnFromTable()) { - return false; - } - if (!(orderKey.getExpr() instanceof SlotReference)) { - return false; - } - SlotReference slotRef = (SlotReference) orderKey.getExpr(); - // do not support alias in project now - if (!t.child().getProjects().contains(slotRef)) { - return false; - } - } - return true; - }) - ).when(project -> project.canMergeChildProjections(project.child().child()))).then(r -> { - LogicalProject upperProject = r.child(); - LogicalProject> bottomProject = r.child().child().child(); - Optional> projections = upperProject.mergeProjections(bottomProject); - if (!projections.isPresent()) { - return null; - } - LogicalProject project = upperProject.withProjects(projections.get()); - LogicalFilter filter = bottomProject.child(); - return deferMaterialize(r, r.child().child(), Optional.of(project), - Optional.of(filter), filter.child()); - }) - ) - ); - } - - private Plan deferMaterialize(LogicalResultSink logicalResultSink, - LogicalTopN logicalTopN, Optional> logicalProject, - Optional> logicalFilter, LogicalOlapScan logicalOlapScan) { - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableTopnLazyMaterialization()) { - return null; - } - IdGenerator exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator(); - Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); - SlotReference columnId = SlotReference.fromColumn( - exprIdGenerator.getNextId(), logicalOlapScan.getTable(), rowId, logicalOlapScan.getQualifier()); - Set orderKeys = Sets.newHashSet(); - Set deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); - logicalFilter.ifPresent(filter -> filter.getConjuncts() - .forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds()))); - logicalTopN.getOrderKeys().stream() - .map(OrderKey::getExpr) - .map(Slot.class::cast) - .peek(orderKeys::add) - .map(NamedExpression::getExprId) - .filter(Objects::nonNull) - .forEach(deferredMaterializedExprIds::remove); - if (logicalProject.isPresent()) { - deferredMaterializedExprIds.retainAll(logicalProject.get().getInputSlots().stream() - .map(NamedExpression::getExprId).collect(Collectors.toSet())); - } - if (deferredMaterializedExprIds.isEmpty()) { - // nothing to deferred materialize - return null; - } - LogicalDeferMaterializeOlapScan deferOlapScan = new LogicalDeferMaterializeOlapScan( - logicalOlapScan, deferredMaterializedExprIds, columnId); - Plan root = logicalFilter.map(f -> f.withChildren(deferOlapScan)).orElse(deferOlapScan); - Set inputSlots = Sets.newHashSet(); - logicalFilter.ifPresent(filter -> inputSlots.addAll(filter.getInputSlots())); - if (logicalProject.isPresent()) { - ImmutableList.Builder requiredSlots = ImmutableList.builder(); - inputSlots.addAll(logicalProject.get().getInputSlots()); - for (Slot output : root.getOutput()) { - if (inputSlots.contains(output) || orderKeys.contains(output)) { - requiredSlots.add(output); - } - } - requiredSlots.add(columnId); - root = new LogicalProject<>(requiredSlots.build(), root); - } - root = new LogicalDeferMaterializeTopN<>((LogicalTopN) logicalTopN.withChildren(root), - deferredMaterializedExprIds, columnId); - if (logicalProject.isPresent()) { - // generate projections with the order exactly same as result output's - Map projectsMap = Maps.newHashMap(); - logicalProject.get().getProjects().forEach(p -> projectsMap.put(p.toSlot(), p)); - List outputProjects = logicalResultSink.getOutput().stream() - .map(projectsMap::get) - .collect(ImmutableList.toImmutableList()); - root = logicalProject.get().withProjectsAndChild(outputProjects, root); - } - root = logicalResultSink.withChildren(root); - return new LogicalDeferMaterializeResultSink<>((LogicalResultSink) root, - logicalOlapScan.getTable(), logicalOlapScan.getSelectedIndexId()); - } - - private long getTopNOptLimitThreshold() { - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) { - if (!ConnectContext.get().getSessionVariable().enableTwoPhaseReadOpt) { - return -1; - } - return ConnectContext.get().getSessionVariable().topnOptLimitThreshold; - } - return -1; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java index 432d25586a0450..86322c99762085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java @@ -74,9 +74,6 @@ public List buildRules() { RuleType.PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N.build( logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDownProject) ), - // RuleType.PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N.build( - // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDownProject) - // ), RuleType.PUSH_DOWN_PROJECT_THROUGH_UNION.build( logicalProject( logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index c90d85d55e667d..bd4831be9d6e63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -38,8 +38,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -231,21 +229,6 @@ public Plan visitLogicalPartitionTopN(LogicalPartitionTopN parti return partitionTopN; } - @Override - public Plan visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, Void context) { - topN = visitChildren(this, topN, context); - - LogicalTopN logicalTopN = (LogicalTopN) topN.getLogicalTopN().accept(this, context); - if (logicalTopN != topN.getLogicalTopN()) { - SlotReference replacedColumnIdSlot = replaceExpressions( - ImmutableList.of(topN.getColumnIdSlot()), false, false).second.get(0); - return new LogicalDeferMaterializeTopN( - logicalTopN, topN.getDeferMaterializeSlotIds(), replacedColumnIdSlot); - } - - return topN; - } - @Override public Plan visitLogicalExcept(LogicalExcept except, Void context) { except = visitChildren(this, except, context); @@ -378,23 +361,6 @@ public Plan visitLogicalTopN(LogicalTopN topN, Void context) { return topN; } - @Override - public Plan visitLogicalDeferMaterializeOlapScan( - LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, Void context) { - - LogicalOlapScan logicalOlapScan - = (LogicalOlapScan) deferMaterializeOlapScan.getLogicalOlapScan().accept(this, context); - - if (logicalOlapScan != deferMaterializeOlapScan.getLogicalOlapScan()) { - SlotReference replacedColumnIdSlot = replaceExpressions( - ImmutableList.of(deferMaterializeOlapScan.getColumnIdSlot()), false, false).second.get(0); - return new LogicalDeferMaterializeOlapScan( - logicalOlapScan, deferMaterializeOlapScan.getDeferMaterializeSlotIds(), replacedColumnIdSlot - ); - } - return deferMaterializeOlapScan; - } - @Override public Plan visitLogicalFilter(LogicalFilter filter, Void context) { filter = visitChildren(this, filter, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java index 7ad4e30bb22005..58ddf9017929f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/StatsDerive.java @@ -29,8 +29,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -181,14 +179,6 @@ public Statistics visitLogicalOlapScan(LogicalOlapScan olapScan, DeriveContext c return stats; } - @Override - public Statistics visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan olapScan, - DeriveContext context) { - Statistics stats = context.calculator.computeOlapScan(olapScan); - olapScan.setStatistics(stats); - return stats; - } - @Override public Statistics visitLogicalCatalogRelation(LogicalCatalogRelation relation, DeriveContext context) { Statistics stats = relation.getStats(); @@ -227,18 +217,6 @@ public Statistics visitLogicalTopN(LogicalTopN topN, DeriveConte return stats; } - @Override - public Statistics visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, - DeriveContext context) { - Statistics stats = topN.getStats(); - if (stats == null && deepDerive) { - Statistics childStats = topN.child().accept(this, context); - stats = context.calculator.computeTopN(topN, childStats); - topN.setStatistics(stats); - } - return stats; - } - @Override public Statistics visitLogicalPartitionTopN(LogicalPartitionTopN partitionTopN, DeriveContext context) { @@ -416,5 +394,3 @@ public Statistics visitLogicalRelation(LogicalRelation relation, DeriveContext c - - diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index f15d53da2457db..b1a8588c0198fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -73,8 +73,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; @@ -107,8 +105,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; @@ -836,12 +832,6 @@ private boolean isVisibleSlotReference(Slot slot) { return false; } - @Override - public Statistics visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, - Void context) { - return computeOlapScan(deferMaterializeOlapScan.getLogicalOlapScan()); - } - @Override public Statistics visitLogicalSchemaScan(LogicalSchemaScan schemaScan, Void context) { return computeCatalogRelation(schemaScan); @@ -919,11 +909,6 @@ public Statistics visitLogicalTopN(LogicalTopN topN, Void contex return computeTopN(topN, groupExpression.childStatistics(0)); } - @Override - public Statistics visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, Void context) { - return computeTopN(topN.getLogicalTopN(), groupExpression.childStatistics(0)); - } - @Override public Statistics visitLogicalPartitionTopN(LogicalPartitionTopN partitionTopN, Void context) { return computePartitionTopN(partitionTopN, groupExpression.childStatistics(0)); @@ -1044,12 +1029,6 @@ public Statistics visitPhysicalOlapScan(PhysicalOlapScan olapScan, Void context) return computeOlapScan(olapScan); } - @Override - public Statistics visitPhysicalDeferMaterializeOlapScan(PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, - Void context) { - return computeCatalogRelation(deferMaterializeOlapScan.getPhysicalOlapScan()); - } - @Override public Statistics visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, Void context) { return computeCatalogRelation(schemaScan); @@ -1122,12 +1101,6 @@ public Statistics visitPhysicalTopN(PhysicalTopN topN, Void cont return computeTopN(topN, groupExpression.childStatistics(0)); } - @Override - public Statistics visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN topN, - Void context) { - return computeTopN(topN.getPhysicalTopN(), groupExpression.childStatistics(0)); - } - @Override public Statistics visitPhysicalHashJoin( PhysicalHashJoin hashJoin, Void context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 0efa3d4e9f17c2..db7be1b7721cee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -38,8 +38,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -48,7 +46,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -232,19 +229,6 @@ public Plan visitLogicalFilter(LogicalFilter filter, DeepCopierC return new LogicalFilter<>(conjuncts, child); } - @Override - public Plan visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, - DeepCopierContext context) { - LogicalOlapScan newScan = (LogicalOlapScan) visitLogicalOlapScan( - deferMaterializeOlapScan.getLogicalOlapScan(), context); - Set newSlotIds = deferMaterializeOlapScan.getDeferMaterializeSlotIds().stream() - .map(context.exprIdReplaceMap::get) - .collect(ImmutableSet.toImmutableSet()); - SlotReference newRowId = (SlotReference) ExpressionDeepCopier.INSTANCE - .deepCopy(deferMaterializeOlapScan.getColumnIdSlot(), context); - return new LogicalDeferMaterializeOlapScan(newScan, newSlotIds, newRowId); - } - @Override public Plan visitLogicalProject(LogicalProject project, DeepCopierContext context) { Plan child = project.child().accept(this, context); @@ -274,19 +258,6 @@ public Plan visitLogicalTopN(LogicalTopN topN, DeepCopierContext return new LogicalTopN<>(orderKeys, topN.getLimit(), topN.getOffset(), child); } - @Override - public Plan visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, - DeepCopierContext context) { - LogicalTopN newTopN - = (LogicalTopN) visitLogicalTopN(topN.getLogicalTopN(), context); - Set newSlotIds = topN.getDeferMaterializeSlotIds().stream() - .map(context.exprIdReplaceMap::get) - .collect(ImmutableSet.toImmutableSet()); - SlotReference newRowId = (SlotReference) ExpressionDeepCopier.INSTANCE - .deepCopy(topN.getColumnIdSlot(), context); - return new LogicalDeferMaterializeTopN<>(newTopN, newSlotIds, newRowId); - } - @Override public Plan visitLogicalPartitionTopN(LogicalPartitionTopN partitionTopN, DeepCopierContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java deleted file mode 100644 index 33741e7fdb2bd3..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java +++ /dev/null @@ -1,175 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.logical; - -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.algebra.OlapScan; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * use for defer materialize top n - */ -public class LogicalDeferMaterializeOlapScan extends LogicalCatalogRelation implements OlapScan { - - private final LogicalOlapScan logicalOlapScan; - - /////////////////////////////////////////////////////////////////////////// - // Members for defer materialize for top-n opt. - /////////////////////////////////////////////////////////////////////////// - private final Set deferMaterializeSlotIds; - private final SlotReference columnIdSlot; - - public LogicalDeferMaterializeOlapScan(LogicalOlapScan logicalOlapScan, - Set deferMaterializeSlotIds, SlotReference columnIdSlot) { - this(logicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - logicalOlapScan.getGroupExpression(), Optional.empty()); - } - - /** - * constructor - */ - public LogicalDeferMaterializeOlapScan(LogicalOlapScan logicalOlapScan, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, Optional logicalProperties) { - super(logicalOlapScan.getRelationId(), logicalOlapScan.getType(), logicalOlapScan.getTable(), - logicalOlapScan.getQualifier(), groupExpression, logicalProperties); - this.logicalOlapScan = Objects.requireNonNull(logicalOlapScan, "logicalOlapScan can not be null"); - this.deferMaterializeSlotIds = ImmutableSet.copyOf(Objects.requireNonNull(deferMaterializeSlotIds, - "deferMaterializeSlotIds can not be null")); - this.columnIdSlot = Objects.requireNonNull(columnIdSlot, "columnIdSlot can not be null"); - } - - public LogicalOlapScan getLogicalOlapScan() { - return logicalOlapScan; - } - - public Set getDeferMaterializeSlotIds() { - return deferMaterializeSlotIds; - } - - public SlotReference getColumnIdSlot() { - return columnIdSlot; - } - - @Override - public OlapTable getTable() { - return logicalOlapScan.getTable(); - } - - @Override - public long getSelectedIndexId() { - return logicalOlapScan.getSelectedIndexId(); - } - - @Override - public List getSelectedPartitionIds() { - return logicalOlapScan.getSelectedPartitionIds(); - } - - @Override - public List getSelectedTabletIds() { - return logicalOlapScan.getSelectedPartitionIds(); - } - - @Override - public List computeOutput() { - return ImmutableList.builder() - .addAll(logicalOlapScan.getOutput()) - .add(columnIdSlot) - .build(); - } - - @Override - public Plan withGroupExpression(Optional groupExpression) { - return new LogicalDeferMaterializeOlapScan(logicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - groupExpression, Optional.of(getLogicalProperties())); - } - - @Override - public Plan withGroupExprLogicalPropChildren(Optional groupExpression, - Optional logicalProperties, List children) { - Preconditions.checkArgument(children.isEmpty(), "LogicalDeferMaterializeOlapScan should have no child"); - return new LogicalDeferMaterializeOlapScan(logicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - groupExpression, logicalProperties); - } - - @Override - public Plan withChildren(List children) { - Preconditions.checkArgument(children.isEmpty(), "LogicalDeferMaterializeOlapScan should have no child"); - return this; - } - - @Override - public LogicalDeferMaterializeOlapScan withRelationId(RelationId relationId) { - throw new RuntimeException("should not call LogicalDeferMaterializeOlapScan's withRelationId method"); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalDeferMaterializeOlapScan(this, context); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - LogicalDeferMaterializeOlapScan that = (LogicalDeferMaterializeOlapScan) o; - return Objects.equals(logicalOlapScan, that.logicalOlapScan) && Objects.equals( - deferMaterializeSlotIds, that.deferMaterializeSlotIds) && Objects.equals(columnIdSlot, - that.columnIdSlot); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), logicalOlapScan, deferMaterializeSlotIds, columnIdSlot); - } - - @Override - public String toString() { - return Utils.toSqlStringSkipNull("LogicalDeferMaterializeOlapScan[" + id.asInt() + "]", - "olapScan", logicalOlapScan, - "deferMaterializeSlotIds", deferMaterializeSlotIds, - "columnIdSlot", columnIdSlot, - "stats", statistics - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java deleted file mode 100644 index 4944b38d8082e0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java +++ /dev/null @@ -1,147 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.logical; - -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; -import org.apache.doris.nereids.trees.plans.algebra.Sink; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * use for defer materialize top n - */ -public class LogicalDeferMaterializeResultSink - extends LogicalSink implements Sink, PropagateFuncDeps { - - private final LogicalResultSink logicalResultSink; - private final OlapTable olapTable; - private final long selectedIndexId; - - public LogicalDeferMaterializeResultSink(LogicalResultSink logicalResultSink, - OlapTable olapTable, long selectedIndexId) { - this(logicalResultSink, olapTable, selectedIndexId, - Optional.empty(), Optional.empty(), logicalResultSink.child()); - } - - public LogicalDeferMaterializeResultSink(LogicalResultSink logicalResultSink, - OlapTable olapTable, long selectedIndexId, - Optional groupExpression, Optional logicalProperties, - CHILD_TYPE child) { - super(logicalResultSink.getType(), logicalResultSink.getOutputExprs(), - groupExpression, logicalProperties, child); - this.logicalResultSink = logicalResultSink; - this.olapTable = olapTable; - this.selectedIndexId = selectedIndexId; - } - - public LogicalResultSink getLogicalResultSink() { - return logicalResultSink; - } - - public OlapTable getOlapTable() { - return olapTable; - } - - public long getSelectedIndexId() { - return selectedIndexId; - } - - @Override - public LogicalDeferMaterializeResultSink withChildren(List children) { - Preconditions.checkArgument(children.size() == 1, - "LogicalDeferMaterializeResultSink only accepts one child"); - return new LogicalDeferMaterializeResultSink<>( - logicalResultSink.withChildren(ImmutableList.of(children.get(0))), - olapTable, selectedIndexId, Optional.empty(), Optional.empty(), children.get(0)); - } - - @Override - public LogicalDeferMaterializeResultSink withOutputExprs(List outputExprs) { - return new LogicalDeferMaterializeResultSink<>(logicalResultSink, olapTable, selectedIndexId, - Optional.empty(), Optional.empty(), child()); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalDeferMaterializeResultSink(this, context); - } - - @Override - public List getExpressions() { - return logicalResultSink.getExpressions(); - } - - @Override - public LogicalDeferMaterializeResultSink withGroupExpression(Optional groupExpression) { - return new LogicalDeferMaterializeResultSink<>(logicalResultSink, olapTable, selectedIndexId, - groupExpression, Optional.of(getLogicalProperties()), child()); - } - - @Override - public LogicalDeferMaterializeResultSink withGroupExprLogicalPropChildren( - Optional groupExpression, - Optional logicalProperties, - List children) { - Preconditions.checkArgument(children.size() == 1, - "LogicalDeferMaterializeResultSink only accepts one child"); - return new LogicalDeferMaterializeResultSink<>( - logicalResultSink.withChildren(ImmutableList.of(children.get(0))), - olapTable, selectedIndexId, groupExpression, logicalProperties, children.get(0)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - LogicalDeferMaterializeResultSink that = (LogicalDeferMaterializeResultSink) o; - return selectedIndexId == that.selectedIndexId && Objects.equals(logicalResultSink, - that.logicalResultSink) && Objects.equals(olapTable, that.olapTable); - } - - @Override - public int hashCode() { - return Objects.hash(logicalResultSink, olapTable, selectedIndexId); - } - - @Override - public String toString() { - return Utils.toSqlStringSkipNull("LogicalDeferMaterializeResultSink[" + id.asInt() + "]", - "logicalResultSink", logicalResultSink, - "olapTable", olapTable, - "selectedIndexId", selectedIndexId, - "stats", statistics - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java deleted file mode 100644 index 8e59c3f8afda67..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java +++ /dev/null @@ -1,191 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.logical; - -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DataTrait; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; -import org.apache.doris.nereids.trees.plans.ObjectId; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.TopN; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * use for defer materialize top n - */ -public class LogicalDeferMaterializeTopN extends LogicalUnary - implements TopN, BlockFuncDepsPropagation { - - private final LogicalTopN logicalTopN; - - /////////////////////////////////////////////////////////////////////////// - // Members for defer materialize for top-n opt. - /////////////////////////////////////////////////////////////////////////// - private final Set deferMaterializeSlotIds; - private final SlotReference columnIdSlot; - - public LogicalDeferMaterializeTopN(LogicalTopN logicalTopN, - Set deferMaterializeSlotIds, SlotReference columnIdSlot) { - super(PlanType.LOGICAL_TOP_N, logicalTopN.getGroupExpression(), - Optional.of(logicalTopN.getLogicalProperties()), logicalTopN.child()); - this.logicalTopN = logicalTopN; - this.deferMaterializeSlotIds = deferMaterializeSlotIds; - this.columnIdSlot = columnIdSlot; - } - - public LogicalDeferMaterializeTopN(LogicalTopN logicalTopN, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, Optional logicalProperties, - CHILD_TYPE child) { - super(PlanType.LOGICAL_TOP_N, groupExpression, logicalProperties, child); - this.logicalTopN = logicalTopN; - this.deferMaterializeSlotIds = deferMaterializeSlotIds; - this.columnIdSlot = columnIdSlot; - } - - public LogicalTopN getLogicalTopN() { - return logicalTopN; - } - - public Set getDeferMaterializeSlotIds() { - return deferMaterializeSlotIds; - } - - public SlotReference getColumnIdSlot() { - return columnIdSlot; - } - - @Override - public List getOrderKeys() { - return logicalTopN.getOrderKeys(); - } - - @Override - public long getOffset() { - return logicalTopN.getOffset(); - } - - @Override - public long getLimit() { - return logicalTopN.getLimit(); - } - - @Override - public List getExpressions() { - return ImmutableList.builder() - .addAll(logicalTopN.getExpressions()) - .add(columnIdSlot).build(); - } - - @Override - public List computeOutput() { - return logicalTopN.getOutput().stream() - .filter(s -> !(s.getExprId().equals(columnIdSlot.getExprId()))) - .collect(ImmutableList.toImmutableList()); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalDeferMaterializeTopN(this, context); - } - - @Override - public Plan withGroupExpression(Optional groupExpression) { - return new LogicalDeferMaterializeTopN<>(logicalTopN, deferMaterializeSlotIds, columnIdSlot, - groupExpression, Optional.of(getLogicalProperties()), child()); - } - - @Override - public Plan withGroupExprLogicalPropChildren(Optional groupExpression, - Optional logicalProperties, List children) { - Preconditions.checkArgument(children.size() == 1, - "LogicalDeferMaterializeTopN should have 1 child, but input is %s", children.size()); - return new LogicalDeferMaterializeTopN<>(logicalTopN.withChildren(ImmutableList.of(children.get(0))), - deferMaterializeSlotIds, columnIdSlot, groupExpression, logicalProperties, children.get(0)); - } - - @Override - public Plan withChildren(List children) { - Preconditions.checkArgument(children.size() == 1, - "LogicalDeferMaterializeTopN should have 1 child, but input is %s", children.size()); - return new LogicalDeferMaterializeTopN<>(logicalTopN.withChildren(ImmutableList.of(children.get(0))), - deferMaterializeSlotIds, columnIdSlot, Optional.empty(), Optional.empty(), children.get(0)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - LogicalDeferMaterializeTopN that = (LogicalDeferMaterializeTopN) o; - return Objects.equals(logicalTopN, that.logicalTopN) && Objects.equals(deferMaterializeSlotIds, - that.deferMaterializeSlotIds) && Objects.equals(columnIdSlot, that.columnIdSlot); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), logicalTopN, deferMaterializeSlotIds, columnIdSlot); - } - - @Override - public String toString() { - return Utils.toSqlStringSkipNull("LogicalDeferMaterializeTopN[" + id.asInt() + "]", - "logicalTopN", logicalTopN, - "deferMaterializeSlotIds", deferMaterializeSlotIds, - "columnIdSlot", columnIdSlot, - "stats", statistics - ); - } - - @Override - public void computeFd(DataTrait.Builder builder) { - builder.addFuncDepsDG(child().getLogicalProperties().getTrait()); - } - - @Override - public void computeEqualSet(DataTrait.Builder builder) { - builder.addEqualSet(child().getLogicalProperties().getTrait()); - } - - @Override - public ObjectId getObjectId() { - return id; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java deleted file mode 100644 index fdf88955fec330..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java +++ /dev/null @@ -1,160 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.physical; - -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.OlapScan; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; -import org.apache.doris.statistics.Statistics; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * use for defer materialize top n - */ -public class PhysicalDeferMaterializeOlapScan extends PhysicalCatalogRelation implements OlapScan { - - private final PhysicalOlapScan physicalOlapScan; - - /////////////////////////////////////////////////////////////////////////// - // Members for defer materialize for top-n opt. - /////////////////////////////////////////////////////////////////////////// - private final Set deferMaterializeSlotIds; - private final SlotReference columnIdSlot; - - public PhysicalDeferMaterializeOlapScan(PhysicalOlapScan physicalOlapScan, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, LogicalProperties logicalProperties) { - this(physicalOlapScan, deferMaterializeSlotIds, columnIdSlot, groupExpression, logicalProperties, null, null); - } - - /** - * constructor - */ - public PhysicalDeferMaterializeOlapScan(PhysicalOlapScan physicalOlapScan, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, LogicalProperties logicalProperties, - PhysicalProperties physicalProperties, Statistics statistics) { - super(physicalOlapScan.getRelationId(), physicalOlapScan.getType(), - physicalOlapScan.getTable(), physicalOlapScan.getQualifier(), - groupExpression, logicalProperties, physicalProperties, statistics, - ImmutableList.of()); - this.physicalOlapScan = physicalOlapScan; - this.deferMaterializeSlotIds = deferMaterializeSlotIds; - this.columnIdSlot = columnIdSlot; - } - - public PhysicalOlapScan getPhysicalOlapScan() { - return physicalOlapScan; - } - - public Set getDeferMaterializeSlotIds() { - return deferMaterializeSlotIds; - } - - public SlotReference getColumnIdSlot() { - return columnIdSlot; - } - - @Override - public OlapTable getTable() { - return physicalOlapScan.getTable(); - } - - @Override - public long getSelectedIndexId() { - return physicalOlapScan.getSelectedIndexId(); - } - - @Override - public List getSelectedPartitionIds() { - return physicalOlapScan.getSelectedPartitionIds(); - } - - @Override - public List getSelectedTabletIds() { - return physicalOlapScan.getSelectedTabletIds(); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitPhysicalDeferMaterializeOlapScan(this, context); - } - - @Override - public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalDeferMaterializeOlapScan(physicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - groupExpression, getLogicalProperties(), physicalProperties, statistics); - } - - @Override - public Plan withGroupExprLogicalPropChildren(Optional groupExpression, - Optional logicalProperties, List children) { - return new PhysicalDeferMaterializeOlapScan(physicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - groupExpression, logicalProperties.get(), physicalProperties, statistics); - } - - @Override - public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalDeferMaterializeOlapScan(physicalOlapScan, deferMaterializeSlotIds, columnIdSlot, - groupExpression, getLogicalProperties(), physicalProperties, statistics); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - PhysicalDeferMaterializeOlapScan that = (PhysicalDeferMaterializeOlapScan) o; - return Objects.equals(physicalOlapScan, that.physicalOlapScan) && Objects.equals( - deferMaterializeSlotIds, that.deferMaterializeSlotIds) && Objects.equals(columnIdSlot, - that.columnIdSlot); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), physicalOlapScan, deferMaterializeSlotIds, columnIdSlot); - } - - @Override - public String toString() { - return Utils.toSqlString("PhysicalDeferMaterializeOlapScan[" + id.asInt() + "]", - "physicalOlapScan", physicalOlapScan, - "deferMaterializeSlotIds", deferMaterializeSlotIds, - "columnIdSlot", columnIdSlot - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeResultSink.java deleted file mode 100644 index 4bb7a5796412ac..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeResultSink.java +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.physical; - -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.Sink; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; -import org.apache.doris.statistics.Statistics; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import org.jetbrains.annotations.Nullable; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * use for defer materialize top n - */ -public class PhysicalDeferMaterializeResultSink - extends PhysicalSink implements Sink { - - private final PhysicalResultSink physicalResultSink; - private final OlapTable olapTable; - private final long selectedIndexId; - - public PhysicalDeferMaterializeResultSink(PhysicalResultSink physicalResultSink, - OlapTable olapTable, long selectedIndexId, - Optional groupExpression, LogicalProperties logicalProperties, - CHILD_TYPE child) { - this(physicalResultSink, olapTable, selectedIndexId, - groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); - } - - public PhysicalDeferMaterializeResultSink(PhysicalResultSink physicalResultSink, - OlapTable olapTable, long selectedIndexId, - Optional groupExpression, LogicalProperties logicalProperties, - @Nullable PhysicalProperties physicalProperties, Statistics statistics, - CHILD_TYPE child) { - super(physicalResultSink.getType(), physicalResultSink.outputExprs, - groupExpression, logicalProperties, physicalProperties, statistics, child); - this.physicalResultSink = physicalResultSink; - this.olapTable = olapTable; - this.selectedIndexId = selectedIndexId; - } - - public PhysicalResultSink getPhysicalResultSink() { - return physicalResultSink; - } - - public OlapTable getOlapTable() { - return olapTable; - } - - public long getSelectedIndexId() { - return selectedIndexId; - } - - @Override - public Plan withChildren(List children) { - Preconditions.checkArgument(children.size() == 1, - "PhysicalDeferMaterializeResultSink's children size must be 1, but real is %s", children.size()); - return new PhysicalDeferMaterializeResultSink<>( - physicalResultSink.withChildren(ImmutableList.of(children.get(0))), - olapTable, selectedIndexId, groupExpression, getLogicalProperties(), - physicalProperties, statistics, children.get(0)); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitPhysicalDeferMaterializeResultSink(this, context); - } - - @Override - public List getExpressions() { - return physicalResultSink.getExpressions(); - } - - @Override - public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalDeferMaterializeResultSink<>(physicalResultSink, olapTable, selectedIndexId, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); - } - - @Override - public Plan withGroupExprLogicalPropChildren(Optional groupExpression, - Optional logicalProperties, List children) { - Preconditions.checkArgument(children.size() == 1, - "PhysicalDeferMaterializeResultSink's children size must be 1, but real is %s", children.size()); - return new PhysicalDeferMaterializeResultSink<>( - physicalResultSink.withChildren(ImmutableList.of(children.get(0))), - olapTable, selectedIndexId, groupExpression, logicalProperties.get(), - physicalProperties, statistics, children.get(0)); - } - - @Override - public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalDeferMaterializeResultSink<>(physicalResultSink, olapTable, selectedIndexId, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); - } - - @Override - public PhysicalDeferMaterializeResultSink resetLogicalProperties() { - return new PhysicalDeferMaterializeResultSink<>(physicalResultSink, olapTable, selectedIndexId, - groupExpression, null, physicalProperties, statistics, child()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PhysicalDeferMaterializeResultSink that = (PhysicalDeferMaterializeResultSink) o; - return selectedIndexId == that.selectedIndexId && physicalResultSink.equals(that.physicalResultSink) - && olapTable.equals(that.olapTable); - } - - @Override - public int hashCode() { - return Objects.hash(physicalResultSink, olapTable, selectedIndexId); - } - - @Override - public String toString() { - return Utils.toSqlString("PhysicalDeferMaterializeResultSink[" + id.asInt() + "]", - "physicalResultSink", physicalResultSink, - "olapTable", olapTable, - "selectedIndexId", selectedIndexId - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java deleted file mode 100644 index 14412f2cfc62df..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java +++ /dev/null @@ -1,213 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.physical; - -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DataTrait; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.TopN; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; -import org.apache.doris.statistics.Statistics; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * use for defer materialize top n - */ -public class PhysicalDeferMaterializeTopN - extends AbstractPhysicalSort implements TopN { - - private final PhysicalTopN physicalTopN; - - /////////////////////////////////////////////////////////////////////////// - // Members for defer materialize for top-n opt. - /////////////////////////////////////////////////////////////////////////// - private final Set deferMaterializeSlotIds; - private final SlotReference columnIdSlot; - - public PhysicalDeferMaterializeTopN(PhysicalTopN physicalTopN, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(physicalTopN, deferMaterializeSlotIds, columnIdSlot, - groupExpression, logicalProperties, null, null, child); - } - - public PhysicalDeferMaterializeTopN(PhysicalTopN physicalTopN, - Set deferMaterializeSlotIds, SlotReference columnIdSlot, - Optional groupExpression, LogicalProperties logicalProperties, - PhysicalProperties physicalProperties, Statistics statistics, CHILD_TYPE child) { - super(physicalTopN.getType(), physicalTopN.getOrderKeys(), physicalTopN.getSortPhase(), - groupExpression, logicalProperties, physicalProperties, statistics, child); - this.physicalTopN = physicalTopN; - this.deferMaterializeSlotIds = deferMaterializeSlotIds; - this.columnIdSlot = columnIdSlot; - } - - public PhysicalTopN getPhysicalTopN() { - return physicalTopN; - } - - public Set getDeferMaterializeSlotIds() { - return deferMaterializeSlotIds; - } - - public SlotReference getColumnIdSlot() { - return columnIdSlot; - } - - @Override - public long getOffset() { - return physicalTopN.getOffset(); - } - - @Override - public long getLimit() { - return physicalTopN.getLimit(); - } - - public PhysicalDeferMaterializeTopN withPhysicalTopN(PhysicalTopN physicalTopN) { - return new PhysicalDeferMaterializeTopN<>(physicalTopN, deferMaterializeSlotIds, columnIdSlot, groupExpression, - getLogicalProperties(), physicalProperties, statistics, physicalTopN.child()); - } - - @Override - public PhysicalDeferMaterializeTopN withChildren(List children) { - Preconditions.checkArgument(children.size() == 1, - "PhysicalDeferMaterializeTopN's children size must be 1, but real is %s", children.size()); - return new PhysicalDeferMaterializeTopN<>(physicalTopN.withChildren(ImmutableList.of(children.get(0))), - deferMaterializeSlotIds, columnIdSlot, groupExpression, getLogicalProperties(), - physicalProperties, statistics, children.get(0)); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitPhysicalDeferMaterializeTopN(this, context); - } - - @Override - public PhysicalDeferMaterializeTopN withGroupExpression(Optional groupExpression) { - return new PhysicalDeferMaterializeTopN<>(physicalTopN, deferMaterializeSlotIds, columnIdSlot, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); - } - - @Override - public PhysicalDeferMaterializeTopN withGroupExprLogicalPropChildren( - Optional groupExpression, - Optional logicalProperties, List children) { - Preconditions.checkArgument(children.size() == 1, - "PhysicalDeferMaterializeTopN's children size must be 1, but real is %s", children.size()); - return new PhysicalDeferMaterializeTopN<>(physicalTopN.withChildren(ImmutableList.of(children.get(0))), - deferMaterializeSlotIds, columnIdSlot, groupExpression, logicalProperties.get(), - physicalProperties, statistics, children.get(0)); - } - - @Override - public PhysicalDeferMaterializeTopN withPhysicalPropertiesAndStats( - PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalDeferMaterializeTopN<>(physicalTopN, deferMaterializeSlotIds, columnIdSlot, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); - } - - @Override - public List computeOutput() { - return child().getOutput(); - } - - @Override - public PhysicalDeferMaterializeTopN resetLogicalProperties() { - return new PhysicalDeferMaterializeTopN<>(physicalTopN, deferMaterializeSlotIds, columnIdSlot, - groupExpression, null, physicalProperties, statistics, child()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - PhysicalDeferMaterializeTopN that = (PhysicalDeferMaterializeTopN) o; - return Objects.equals(physicalTopN, that.physicalTopN) && Objects.equals( - deferMaterializeSlotIds, that.deferMaterializeSlotIds) && Objects.equals(columnIdSlot, - that.columnIdSlot); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), physicalTopN, deferMaterializeSlotIds, columnIdSlot); - } - - @Override - public String toString() { - return Utils.toSqlString("PhysicalDeferMaterializeTopN[" + id.asInt() + "]", - "physicalTopN", physicalTopN, - "deferMaterializeSlotIds", deferMaterializeSlotIds, - "columnIdSlot", columnIdSlot - ); - } - - @Override - public void computeUnique(DataTrait.Builder builder) { - if (getLimit() == 1) { - getOutput().forEach(builder::addUniqueSlot); - } else { - builder.addUniqueSlot(child(0).getLogicalProperties().getTrait()); - } - } - - @Override - public void computeUniform(DataTrait.Builder builder) { - if (getLimit() == 1) { - getOutput().forEach(builder::addUniformSlot); - } else { - builder.addUniformSlot(child(0).getLogicalProperties().getTrait()); - } - } - - @Override - public void computeFd(DataTrait.Builder builder) { - builder.addFuncDepsDG(child().getLogicalProperties().getTrait()); - } - - @Override - public void computeEqualSet(DataTrait.Builder builder) { - builder.addEqualSet(child().getLogicalProperties().getTrait()); - } - - @Override - public ObjectId getObjectId() { - return id; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index 1aa50b53c05a36..55b677923c5e66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; @@ -65,7 +64,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -297,10 +295,6 @@ public R visitLogicalTopN(LogicalTopN topN, C context) { return visit(topN, context); } - public R visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN topN, C context) { - return visit(topN, context); - } - public R visitPhysicalLazyMaterialize(PhysicalLazyMaterialize materialize, C context) { return visit(materialize, context); } @@ -429,10 +423,6 @@ public R visitPhysicalTopN(PhysicalTopN topN, C context) { return visitAbstractPhysicalSort(topN, context); } - public R visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN topN, C context) { - return visitAbstractPhysicalSort(topN, context); - } - public R visitPhysicalWindow(PhysicalWindow window, C context) { return visit(window, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index d9593ee12fe1c5..92e39a781ab216 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -37,7 +36,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalWorkTableReference; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; @@ -120,11 +118,6 @@ default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { return visitLogicalCatalogRelation(olapScan, context); } - default R visitLogicalDeferMaterializeOlapScan( - LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, C context) { - return visitLogicalCatalogRelation(deferMaterializeOlapScan, context); - } - default R visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, C context) { return visitLogicalRelation(oneRowRelation, context); } @@ -177,11 +170,6 @@ default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) { return visitPhysicalCatalogRelation(olapScan, context); } - default R visitPhysicalDeferMaterializeOlapScan( - PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, C context) { - return visitPhysicalCatalogRelation(deferMaterializeOlapScan, context); - } - default R visitPhysicalWorkTableReference(PhysicalWorkTableReference workTableReference, C context) { return visitPhysicalRelation(workTableReference, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java index b02ea52b2a88f1..aa198be66b03e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalBlackholeSink; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalDictionarySink; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalHiveTableSink; @@ -45,7 +44,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalBlackholeSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDictionarySink; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; @@ -170,11 +168,6 @@ default R visitLogicalDictionarySink(LogicalDictionarySink logic return visitLogicalTableSink(logicalDictionarySink, context); } - default R visitLogicalDeferMaterializeResultSink( - LogicalDeferMaterializeResultSink logicalDeferMaterializeResultSink, C context) { - return visitLogicalSink(logicalDeferMaterializeResultSink, context); - } - default R visitLogicalBlackholeSink( LogicalBlackholeSink logicalBlackholeSink, C context) { return visitLogicalSink(logicalBlackholeSink, context); @@ -242,11 +235,6 @@ default R visitPhysicalResultSink(PhysicalResultSink physicalRes return visitPhysicalSink(physicalResultSink, context); } - default R visitPhysicalDeferMaterializeResultSink( - PhysicalDeferMaterializeResultSink sink, C context) { - return visitPhysicalSink(sink, context); - } - default R visitPhysicalTVFTableSink( PhysicalTVFTableSink tvfTableSink, C context) { return visitPhysicalSink(tvfTableSink, context); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index 3a10a7dc1621bc..50fab95e69cb8e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -18,7 +18,6 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.IndexDef; -import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.cloud.common.util.CloudPropertyAnalyzer; import org.apache.doris.cloud.proto.Cloud; @@ -30,13 +29,8 @@ import org.apache.doris.common.util.UnitTestUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; -import org.apache.doris.resource.Tag; -import org.apache.doris.resource.computegroup.ComputeGroup; -import org.apache.doris.system.Backend; -import org.apache.doris.thrift.TFetchOption; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; -import org.apache.doris.utframe.UtFrameUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -435,37 +429,6 @@ public List getVisibleIndex() { Assert.assertTrue(schemaAllIndexes.contains(col2)); } - @Test - public void testTopNPushDownWithTag() throws Exception { - FeConstants.runningUnitTest = true; - - Tag taga = Tag.create(Tag.TYPE_LOCATION, "taga"); - Backend be1 = new Backend(10001, "192.168.1.1", 9050); - be1.setTagMap(taga.toMap()); - be1.setAlive(true); - - Tag tagb = Tag.create(Tag.TYPE_LOCATION, "tagb"); - Backend be2 = new Backend(10002, "192.168.1.2", 9050); - be2.setAlive(true); - be2.setTagMap(tagb.toMap()); - - Env.getCurrentSystemInfo().addBackend(be1); - Env.getCurrentSystemInfo().addBackend(be2); - - ConnectContext connectContext = UtFrameUtils.createDefaultCtx(); - connectContext.setCurrentUserIdentity(UserIdentity.ROOT); - OlapTable tab = new OlapTable(); - TFetchOption tfetchOption = tab.generateTwoPhaseReadOption(-1); - Assert.assertTrue(tfetchOption.nodes_info.nodes.size() == 2); - - connectContext.setComputeGroup(new ComputeGroup("taga", "taga", Env.getCurrentSystemInfo())); - - TFetchOption tfetchOption2 = tab.generateTwoPhaseReadOption(-1); - Assert.assertTrue(tfetchOption2.nodes_info.nodes.size() == 1); - ConnectContext.remove(); - - } - @Test public void testTableVersionCacheWithRpc() throws Exception { // Mock cloud mode diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKeyTest.java index a54d65d9fcb202..016fc6886a8fc4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKeyTest.java @@ -36,7 +36,7 @@ protected void runBeforeAll() throws Exception { createTable("create table test.test_unique_order_by2(a int not null, b int not null, c int, d int) " + "unique key(a,b) distributed by hash(a) properties('replication_num'='1');"); connectContext.setDatabase("test"); - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION,DEFER_MATERIALIZE_TOP_N_RESULT"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); } @Test diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index a4e2cda7440bf3..fedb2a20d0f1a1 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -217,6 +217,7 @@ struct TMultiCastDataStreamSink { 2: optional list> destinations; } +// [deprecated] two phase read is replaced by topn lazy materialize. TFetchOption is not used. struct TFetchOption { 1: optional bool use_two_phase_fetch; // Nodes in this cluster, used for second phase fetch diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy deleted file mode 100644 index de8fd90bbfa901..00000000000000 --- a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -suite("lazy_materialize_topn") { - sql """ - set enable_two_phase_read_opt = true; - set topn_opt_limit_threshold = 1000; - set topn_lazy_materialization_threshold = -1; - """ - - sql """ - drop table if exists lazy_materialize_topn; - """ - - sql """ - CREATE TABLE `lazy_materialize_topn` ( - `c1` int NULL, - `c2` int NULL, - `c3` int NULL, - `c4` array NULL - ) - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "light_schema_change" = "true" - ); - """ - - sql """ - insert into lazy_materialize_topn values (1, 1, 1, [1]), (2, 2, 2, [2]), (3, 3, 3, [3]); - """ - - sql """ - sync - """ - - List sqls = [ - // TopN(Scan) - """select * from lazy_materialize_topn order by c1 limit 10""", - // TopN(Project(Scan)) - """select c1, c2 from lazy_materialize_topn order by c1 limit 10""", - // Project(TopN(Scan)) - """select c1, c2, c3, c4 from lazy_materialize_topn order by c1 limit 10""", - // Project(TopN(Project(Scan))) - """select c1 + 1, c2 + 1 from (select c1, c2 from lazy_materialize_topn order by c1 limit 10) t""", - // TopN(Filter(Scan)) - """select * from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", - // TopN(Project(Filter(Scan))) - """select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", - // Project(TopN(Project(Filter(Scan)))) - """select c1 + 1, c2 + 1, c3 + 1 from ( select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10) t""", - // project set is diff with output list - """select c1, c1, c2 from (select c1, c2 from lazy_materialize_topn where c3 < 1 order by c2 limit 1)t;""" - ] - - for (sqlStr in sqls) { - explain { - sql """${sqlStr}""" - contains """OPT TWO PHASE""" - } - sql """${sqlStr}""" - } -} diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy deleted file mode 100644 index 23ab6f8467936e..00000000000000 --- a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -suite("one_phase") { - sql """ - drop table if exists users; - - CREATE TABLE `users` ( - `UserID` bigint NULL - ) ENGINE=OLAP - DUPLICATE KEY(`UserID`) - DISTRIBUTED BY HASH(`UserID`) BUCKETS 48 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "min_load_replica_num" = "-1", - "is_being_synced" = "false", - "storage_medium" = "hdd", - "storage_format" = "V2", - "inverted_index_storage_format" = "V2", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "group_commit_interval_ms" = "10000", - "group_commit_data_bytes" = "134217728" - ); - - insert into users values (11111),(11112),(11113); - - """ - - sql "set sort_phase_num=1;" - qt_1 "select userid from users order by userid limit 2, 109000000;" - - sql "set sort_phase_num=2;" - qt_2 "select userid from users order by userid limit 2, 109000000;" - -} \ No newline at end of file diff --git a/regression-test/suites/query_p0/sort/sort.groovy b/regression-test/suites/query_p0/sort/sort.groovy index ecd8581ebbcd6c..ffe83e44418eb8 100644 --- a/regression-test/suites/query_p0/sort/sort.groovy +++ b/regression-test/suites/query_p0/sort/sort.groovy @@ -124,10 +124,6 @@ suite("sort") { sql "insert into sort_default_value values (3, 0)" sql "insert into sort_default_value values (4, null)" qt_sql "select * from sort_default_value order by k1 limit 10" - explain { - sql("select * from sort_default_value order by k1 limit 10") - contains "OPT TWO PHASE" - } def tblName = "sort_one_float_column" sql """ DROP TABLE IF EXISTS ${tblName} """ diff --git a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy deleted file mode 100644 index 4a97b87efe4e75..00000000000000 --- a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("topn_2pr_rule") { - sql """set topn_opt_limit_threshold = 1024""" - sql """set enable_two_phase_read_opt= true""" - // this case is used to test defer materialze, and hence turn topn_lazy_materialization off - sql """set topn_lazy_materialization_threshold=-1;""" - - def create_table = { table_name, key_type="DUPLICATE" -> - sql "DROP TABLE IF EXISTS ${table_name}" - def value_type = "v string" - if ("${key_type}" == "AGGREGATE") { - value_type = "v string REPLACE_IF_NOT_NULL NULL" - } - sql """ - CREATE TABLE IF NOT EXISTS ${table_name} ( - k bigint, - ${value_type} - ) - ${key_type} KEY(`k`) - DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "false"); - """ - } - def verify = { table_name, key_type-> - if("${key_type}" == "DUPLICATE") { - explain { - sql("select * from ${table_name} order by k limit 1;") - contains "OPT TWO PHASE" - } - explain { - sql("select * from ${table_name} where k > 1 order by k limit 1;") - contains "OPT TWO PHASE" - } - explain { - sql("select * from ${table_name} where k order by k + 1 limit 1;") - notContains "OPT TWO PHASE" - } - } else if("${key_type}" == "UNIQUE") { - explain { - sql("select * from ${table_name} order by k limit 1;") - contains "OPT TWO PHASE" - } - } else if("${key_type}" == "AGGREGATE") { - explain { - sql("select * from ${table_name} order by k limit 1;") - notContains "OPT TWO PHASE" - } - } - } - - def key_types = ["DUPLICATE", "UNIQUE", "AGGREGATE"] - for (int i = 0; i < key_types.size(); i++) { - def table_name = "topn_2pr_rule_${key_types[i]}" - create_table.call(table_name, key_types[i]) - sql """insert into ${table_name} values(1, "1")""" - sql """insert into ${table_name} values(2, "2")""" - sql """insert into ${table_name} values(3, "3")""" - verify.call(table_name, key_types[i]) - } -} \ No newline at end of file diff --git a/regression-test/suites/query_p0/topn_lazy/lazy_materialize_view.groovy b/regression-test/suites/query_p0/topn_lazy/lazy_materialize_view.groovy index 8e58ddaa7cbe68..763cf40f3ccf96 100644 --- a/regression-test/suites/query_p0/topn_lazy/lazy_materialize_view.groovy +++ b/regression-test/suites/query_p0/topn_lazy/lazy_materialize_view.groovy @@ -17,9 +17,8 @@ suite("lazy_materialize_view") { sql """ - set enable_two_phase_read_opt = true; set topn_opt_limit_threshold = 1000; - set topn_lazy_materialization_threshold = -1; + set topn_lazy_materialization_threshold = 1024; """ sql "drop table if exists lazy_mat_view_t" @@ -55,7 +54,7 @@ suite("lazy_materialize_view") { // lazy materialization through a single-level view explain { sql "select v1 from lazy_mat_view_v1 order by k1 limit 2" - contains "OPT TWO PHASE" + contains "VMaterializeNode" } order_qt_view_lazy """ @@ -65,7 +64,7 @@ suite("lazy_materialize_view") { // lazy materialization through a chained view (v2 -> v1 -> t) explain { sql "select v1 from lazy_mat_view_v2 order by k1 limit 2" - contains "OPT TWO PHASE" + contains "VMaterializeNode" } order_qt_chain_view_lazy """ @@ -75,7 +74,7 @@ suite("lazy_materialize_view") { // lazy materialization through view with filter explain { sql "select v1, v2 from lazy_mat_view_v1 where k2 > 10 order by k1 limit 2" - contains "OPT TWO PHASE" + contains "VMaterializeNode" } order_qt_view_lazy_filter """