From 4005db369c2c81ce338539784ed758353f49a4e2 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Mon, 24 Aug 2026 15:49:51 +0100 Subject: [PATCH 1/6] HIVE-29734: Cleanup Bug.java after Calcite upgrade 1.42.0 --- data/conf/hive-log4j2.properties | 2 +- .../hadoop/hive/ql/optimizer/calcite/Bug.java | 28 ---- .../ql/optimizer/calcite/HiveRelBuilder.java | 102 +----------- .../ql/optimizer/calcite/RuleEventLogger.java | 89 ---------- .../calcite/rules/HiveFilterJoinRule.java | 153 +----------------- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- 6 files changed, 5 insertions(+), 371 deletions(-) delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java diff --git a/data/conf/hive-log4j2.properties b/data/conf/hive-log4j2.properties index 5eb2480a2a6e..bbcec098dd8c 100644 --- a/data/conf/hive-log4j2.properties +++ b/data/conf/hive-log4j2.properties @@ -114,7 +114,7 @@ logger.CalcitePlanner.name = org.apache.calcite.plan.RelOptPlanner logger.CalcitePlanner.level = INFO # Change logger level to DEBUG, to see all application of CBO rules and the operators they produce -logger.CBORuleLogger.name = org.apache.hadoop.hive.ql.optimizer.calcite.RuleEventLogger +logger.CBORuleLogger.name = org.apache.calcite.plan.RuleEventLogger logger.CBORuleLogger.level = OFF logger.CBORuleLogger.filter.marker.type = MarkerFilter logger.CBORuleLogger.filter.marker.marker = FULL_PLAN diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java index fbd48a721e5a..3df5262739c6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -42,34 +42,6 @@ public final class Bug { */ public static final boolean CALCITE_4166_FIXED = false; - /** - * Whether issue - * CALCITE-4499 is fixed. - */ - public static final boolean CALCITE_4499_FIXED = false; - - /** - * Whether issue - * CALCITE-4200 is fixed. - */ - public static final boolean CALCITE_4200_FIXED = false; - - /** - * Whether issue - * CALCITE-4513 is fixed. - */ - public static final boolean CALCITE_4513_FIXED=false; - - /** - * Whether CALCITE-4574 is fixed. - */ - public static final boolean CALCITE_4574_FIXED = false; - - /** - * Whether CALCITE-4704 is fixed. - */ - public static final boolean CALCITE_4704_FIXED = false; - /** * Whether CALCITE-5293 is fixed. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java index d76c545b3aba..cb80fcaf8396 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java @@ -18,23 +18,13 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite; -import com.google.common.collect.Iterables; import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptSchema; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.server.CalciteServerStatement; @@ -44,14 +34,8 @@ import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Litmus; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - /** * Builder for relational expressions in Hive. * @@ -90,7 +74,7 @@ public static RelBuilderFactory proto(final Context context) { return new RelBuilderFactory() { @Override public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) { - Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED) + Context confContext = Contexts.of(Config.DEFAULT .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema); } @@ -133,88 +117,4 @@ public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) { } return SqlStdOperatorTable.FLOOR; } - - /** Creates a {@link Join} with correlating variables. */ - @Override - public RelBuilder join(JoinRelType joinType, RexNode condition, - Set variablesSet) { - if (Bug.CALCITE_4574_FIXED) { - throw new IllegalStateException("Method overriding should be removed once CALCITE-4574 is fixed"); - } - RelNode right = this.peek(0); - RelNode left = this.peek(1); - final boolean correlate = variablesSet.size() == 1; - RexNode postCondition = literal(true); - if (correlate) { - final CorrelationId id = Iterables.getOnlyElement(variablesSet); - if (!RelOptUtil.notContainsCorrelation(left, id, Litmus.IGNORE)) { - throw new IllegalArgumentException("variable " + id - + " must not be used by left input to correlation"); - } - // Correlate does not have an ON clause. - switch (joinType) { - case LEFT: - case SEMI: - case ANTI: - // For a LEFT/SEMI/ANTI, predicate must be evaluated first. - filter(condition.accept(new Shifter(left, id, right))); - right = this.peek(0); - break; - case INNER: - // For INNER, we can defer. - postCondition = condition; - break; - default: - throw new IllegalArgumentException("Correlated " + joinType + " join is not supported"); - } - final ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, right); - List leftFields = this.fields(2, 0); - List requiredFields = new ArrayList<>(); - for (int i = 0; i < leftFields.size(); i++) { - if (requiredColumns.get(i)) { - requiredFields.add(leftFields.get(i)); - } - } - correlate(joinType, id, requiredFields); - filter(postCondition); - } else { - // When there is no correlation use the default logic which works OK for now - // Cannot copy-paste the respective code here cause we don't have access to stack, - // Frame etc. and we might lose existing aliases in the builder - assert variablesSet.isEmpty(); - super.join(joinType,condition, variablesSet); - } - return this; - } - - /** Shuttle that shifts a predicate's inputs to the left, replacing early - * ones with references to a - * {@link RexCorrelVariable}. */ - private class Shifter extends RexShuttle { - private final RelNode left; - private final CorrelationId id; - private final RelNode right; - - Shifter(RelNode left, CorrelationId id, RelNode right) { - this.left = left; - this.id = id; - this.right = right; - if (Bug.CALCITE_4574_FIXED) { - throw new IllegalStateException("Class should be redundant once CALCITE-4574 is fixed"); - } - } - - public RexNode visitInputRef(RexInputRef inputRef) { - final RelDataType leftRowType = left.getRowType(); - final RexBuilder rexBuilder = getRexBuilder(); - final int leftCount = leftRowType.getFieldCount(); - if (inputRef.getIndex() < leftCount) { - final RexNode v = rexBuilder.makeCorrel(leftRowType, id); - return rexBuilder.makeFieldAccess(v, inputRef.getIndex()); - } else { - return rexBuilder.makeInputRef(right, inputRef.getIndex() - leftCount); - } - } - } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java deleted file mode 100644 index 248fd741abed..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite; - -import org.apache.calcite.plan.RelOptListener; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.Marker; -import org.slf4j.MarkerFactory; - -import java.util.Arrays; -import java.util.stream.Collectors; - -/** - * Listener for logging useful debugging information on certain rule events. - */ -public class RuleEventLogger implements RelOptListener { - private static final Logger LOG = LoggerFactory.getLogger(RuleEventLogger.class.getName()); - private static final Marker FULL = MarkerFactory.getMarker("FULL_PLAN"); - - public RuleEventLogger() { - if (Bug.CALCITE_4704_FIXED) { - throw new IllegalStateException("Class redundant after fix is merged into Calcite"); - } - } - - @Override - public void relEquivalenceFound(final RelEquivalenceEvent event) { - - } - - @Override - public void ruleAttempted(final RuleAttemptedEvent event) { - if (event.isBefore() && LOG.isDebugEnabled()) { - RelOptRuleCall call = event.getRuleCall(); - String ruleArgs = Arrays.stream(call.rels).map(rel -> "rel#" + rel.getId() + ":" + rel.getRelTypeName()) - .collect(Collectors.joining(",")); - LOG.debug("call#{}: Apply rule [{}] to [{}]", call.id, call.getRule(), ruleArgs); - } - } - - @Override - public void ruleProductionSucceeded(RuleProductionEvent event) { - if (event.isBefore() && LOG.isDebugEnabled()) { - RelOptRuleCall call = event.getRuleCall(); - - Arrays.stream(call.rels).forEach(rel -> - LOG.debug(FULL, "call#{}: Full plan for rule input [rel#{}:{}]: {}", call.id, rel.getId(), - rel.getRelTypeName(), System.lineSeparator() + RelOptUtil.toString(rel))); - - RelNode newRel = event.getRel(); - String description = newRel == null ? "null" : "rel#" + newRel.getId() + ":" + newRel.getRelTypeName(); - LOG.debug("call#{}: Rule [{}] produced [{}]", call.id, call.getRule(), description); - if (newRel != null) { - LOG.debug(FULL, "call#{}: Full plan for [{}]:{}", call.id, description, - System.lineSeparator() + RelOptUtil.toString(newRel)); - } - } - } - - @Override - public void relDiscarded(final RelDiscardedEvent event) { - - } - - @Override - public void relChosen(final RelChosenEvent event) { - - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java index ac4ba209fc80..38ee55b3ae95 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java @@ -27,22 +27,17 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.RelOptUtil.InputFinder; -import org.apache.calcite.plan.RelOptUtil.RexInputConverter; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; @@ -209,7 +204,7 @@ protected void perform(RelOptRuleCall call, Filter filter, Join join) { // Try to push down above filters. These are typically where clause // filters. They can be pushed down if they are not on the NULL // generating side. - boolean filterPushed = classifyFilters( + boolean filterPushed = RelOptUtil.classifyFilters( join, aboveFilters, joinType == JoinRelType.INNER || joinType == JoinRelType.SEMI, @@ -250,7 +245,7 @@ protected void perform(RelOptRuleCall call, Filter filter, Join join) { // that is, the result set may be smaller. The right can not be pushed // into for the same reason. if (joinType != JoinRelType.ANTI - && classifyFilters( + && RelOptUtil.classifyFilters( join, joinFilters, false, @@ -341,150 +336,6 @@ private List getConjunctions(Filter filter) { } return conjunctions; } - - /** - * Classifies filters according to where they should be processed. They - * either stay where they are, are pushed to the join (if they originated - * from above the join), or are pushed to one of the children. Filters that - * are pushed are added to list passed in as input parameters. - * - * @param joinRel join node - * @param filters filters to be classified - * @param pushInto whether filters can be pushed into the join - * @param pushLeft true if filters can be pushed to the left - * @param pushRight true if filters can be pushed to the right - * @param joinFilters list of filters to push to the join - * @param leftFilters list of filters to push to the left child - * @param rightFilters list of filters to push to the right child - * @return whether at least one filter was pushed - */ - private static boolean classifyFilters( - RelNode joinRel, - List filters, - boolean pushInto, - boolean pushLeft, - boolean pushRight, - List joinFilters, - List leftFilters, - List rightFilters) { - if (Bug.CALCITE_4499_FIXED) { - throw new AssertionError("Remove this method when [CALCITE-4499] " - + "has been fixed and use directly Calcite's RelOptUtil.classifyFilters."); - } - RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); - List joinFields = joinRel.getRowType().getFieldList(); - final int nSysFields = 0; // joinRel.getSystemFieldList().size(); - final List leftFields = - joinRel.getInputs().get(0).getRowType().getFieldList(); - final int nFieldsLeft = leftFields.size(); - final List rightFields = - joinRel.getInputs().get(1).getRowType().getFieldList(); - final int nFieldsRight = rightFields.size(); - final int nTotalFields = nFieldsLeft + nFieldsRight; - - // set the reference bitmaps for the left and right children - ImmutableBitSet leftBitmap = - ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft); - ImmutableBitSet rightBitmap = - ImmutableBitSet.range(nSysFields + nFieldsLeft, nTotalFields); - - final List filtersToRemove = new ArrayList<>(); - for (RexNode filter : filters) { - final InputFinder inputFinder = InputFinder.analyze(filter); - final ImmutableBitSet inputBits = inputFinder.build(); - - // REVIEW - are there any expressions that need special handling - // and therefore cannot be pushed? - - // filters can be pushed to the left child if the left child - // does not generate NULLs and the only columns referenced in - // the filter originate from the left child - if (pushLeft && leftBitmap.contains(inputBits)) { - // ignore filters that always evaluate to true - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the left now shift over by the number - // of system fields - final RexNode shiftedFilter = - shiftFilter( - nSysFields, - nSysFields + nFieldsLeft, - -nSysFields, - rexBuilder, - joinFields, - nTotalFields, - leftFields, - filter); - - leftFilters.add(shiftedFilter); - } - filtersToRemove.add(filter); - - // filters can be pushed to the right child if the right child - // does not generate NULLs and the only columns referenced in - // the filter originate from the right child - } else if (pushRight && rightBitmap.contains(inputBits)) { - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the right now shift over to the left; - // since we never push filters to a NULL generating - // child, the types of the source should match the dest - // so we don't need to explicitly pass the destination - // fields to RexInputConverter - final RexNode shiftedFilter = - shiftFilter( - nSysFields + nFieldsLeft, - nTotalFields, - -(nSysFields + nFieldsLeft), - rexBuilder, - joinFields, - nTotalFields, - rightFields, - filter); - rightFilters.add(shiftedFilter); - } - filtersToRemove.add(filter); - - } else { - // If the filter can't be pushed to either child, we may push them into the join - if (pushInto) { - if (!joinFilters.contains(filter)) { - joinFilters.add(filter); - } - filtersToRemove.add(filter); - } - } - } - - // Remove filters after the loop, to prevent concurrent modification. - if (!filtersToRemove.isEmpty()) { - filters.removeAll(filtersToRemove); - } - - // Did anything change? - return !filtersToRemove.isEmpty(); - } - - private static RexNode shiftFilter( - int start, - int end, - int offset, - RexBuilder rexBuilder, - List joinFields, - int nTotalFields, - List rightFields, - RexNode filter) { - int[] adjustments = new int[nTotalFields]; - for (int i = start; i < end; i++) { - adjustments[i] = offset; - } - return filter.accept( - new RexInputConverter( - rexBuilder, - joinFields, - rightFields, - adjustments)); - } } // End PushFilterPastJoinRule.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 819ff93dbb58..f370f20c626f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -60,6 +60,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.RuleEventLogger; import org.apache.calcite.plan.hep.HepMatchOrder; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; @@ -168,7 +169,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveMaterializedViewASTSubQueryRewriteShuttle; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveSqlTypeUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.RuleEventLogger; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.CteRuleConfig; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateSortLimitRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinSwapConstraintsRule; From 759753e2fed949f8091426e9a7ba737eced05e30 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 25 Aug 2026 08:52:59 +0100 Subject: [PATCH 2/6] Put back HivelRelBuilder withPruneInputOfAggregate(false) due to tests errors --- .../apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java index cb80fcaf8396..068e56acdfb4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java @@ -74,7 +74,7 @@ public static RelBuilderFactory proto(final Context context) { return new RelBuilderFactory() { @Override public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) { - Context confContext = Contexts.of(Config.DEFAULT + Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(false) // TODO clarify CALCITE_4513 .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema); } From 398a1eeddadbeed2cdd730dbf239ec460d34ad1a Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 25 Aug 2026 13:16:49 +0100 Subject: [PATCH 3/6] More cleanup --- .../hadoop/hive/ql/optimizer/calcite/Bug.java | 25 -- .../HiveFilterTableFunctionTransposeRule.java | 139 +--------- .../rules/HiveRemoveEmptySingleRules.java | 239 ++---------------- .../rules/HiveUnionPullUpConstantsRule.java | 137 +--------- 4 files changed, 39 insertions(+), 501 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java index 3df5262739c6..da7fbfd78aec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -42,31 +42,6 @@ public final class Bug { */ public static final boolean CALCITE_4166_FIXED = false; - /** - * Whether CALCITE-5293 is fixed. - */ - public static final boolean CALCITE_5293_FIXED = false; - - /** - * Whether CALCITE-5294 is fixed. - */ - public static final boolean CALCITE_5294_FIXED = false; - - /** - * Whether CALCITE-5337 is fixed. - */ - public static final boolean CALCITE_5337_FIXED = false; - - /** - * Whether CALCITE-5669 is fixed. - */ - public static final boolean CALCITE_5669_FIXED = false; - - /** - * Whether CALCITE-5985 is fixed. - */ - public static final boolean CALCITE_5985_FIXED = false; - /** * Whether CALCITE-6513 is fixed. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java index ee8562f34362..b4f3f01df96d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java @@ -19,143 +19,26 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.metadata.RelColumnMapping; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; +import org.apache.calcite.rel.rules.FilterTableFunctionTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - /** * Rule to transpose Filter and TableFunctionScan RelNodes - * - * We cannot use Calcite's FilterTableFunctionTransposeRule because that rule - * uses LogicalFilter and LogicalTableFunctionScan. We should remove this - * class when CALCITE-5985 is fixed (and remove the CALCITE_5985_FIXED entry - * in Bug.java) */ -public class HiveFilterTableFunctionTransposeRule extends RelOptRule { - - public static final HiveFilterTableFunctionTransposeRule INSTANCE = - new HiveFilterTableFunctionTransposeRule(HiveRelFactories.HIVE_BUILDER); - - public HiveFilterTableFunctionTransposeRule(RelBuilderFactory relBuilderFactory) { - super(operand(HiveFilter.class, operand(HiveTableFunctionScan.class, any())), - relBuilderFactory, null); - } - - @Override - public boolean matches(RelOptRuleCall call) { - if (Bug.CALCITE_5985_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - - final Filter filterRel = call.rel(0); - final HiveTableFunctionScan tfs = call.rel(1); - - RexNode condition = filterRel.getCondition(); - if (!HiveCalciteUtil.isDeterministic(condition)) { - return false; - } - - // The TableFunctionScan is always created such that all the input RelNode - // fields are present in its RelNode. If a Filter has an InputRef that is - // greater then the number of the RelNode below the TableFunctionScan, that - // means it was a field created by the TableFunctionScan and thus the Filter - // cannot be pushed through. - // - // We check for each individual conjunction (breaking it up by top level 'and' - // conditions). - for (RexNode ce : RelOptUtil.conjunctions(filterRel.getCondition())) { - if (canBePushed(HiveCalciteUtil.getInputRefs(ce), tfs)) { - return true; - } - } - return false; - } +public class HiveFilterTableFunctionTransposeRule { - public void onMatch(RelOptRuleCall call) { - final Filter filter = call.rel(0); - final HiveTableFunctionScan tfs = call.rel(1); - final RelBuilder builder = call.builder(); - - final List newPartKeyFilterConditions = new ArrayList<>(); - final List unpushedFilterConditions = new ArrayList<>(); - - // Check for each individual 'and' condition so that we can push partial - // expressions through. - for (RexNode ce : RelOptUtil.conjunctions(filter.getCondition())) { - // We can only push if all the InputRef pointers are referencing the - // input RelNode to the TableFunctionScan - if (canBePushed(HiveCalciteUtil.getInputRefs(ce), tfs)) { - newPartKeyFilterConditions.add(ce); - } else { - unpushedFilterConditions.add(ce); - } - } - - // The "matches" check should guarantee there's something to push. - final RexNode filterCondToPushBelowProj = RexUtil.composeConjunction( - filter.getCluster().getRexBuilder(), newPartKeyFilterConditions, true); - - builder.push(tfs.getInput(0)).filter(filterCondToPushBelowProj); - - // If there are conditions that cannot be pushed through, generate the RexNode - final RexNode unpushedFilCondAboveProj = unpushedFilterConditions.isEmpty() - ? null - : RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), - unpushedFilterConditions, true); - - // Generate the new TableFunctionScanNode with the Filter InputRel - final RelNode tableFunctionScanNode = tfs.copy(tfs.getTraitSet(), - ImmutableList.of(builder.build()), tfs.getCall(), tfs.getElementType(), - tfs.getRowType(), tfs.getColumnMappings()); - - builder.clear(); - builder.push(tableFunctionScanNode); - - if (unpushedFilCondAboveProj != null) { - builder.filter(unpushedFilCondAboveProj); - } - - call.transformTo(builder.build()); + private HiveFilterTableFunctionTransposeRule() { + throw new IllegalStateException("Instantiation not allowed"); } - // If any of the inputRefs are references to a field that is not mapped into the inputRelNode, - // the condition cannot be pushed. - private boolean canBePushed(Set inputRefs, HiveTableFunctionScan tfs) { - Set columnMappings = tfs.getColumnMappings(); - if (inputRefs.isEmpty()) { - return true; - } + public static final RelOptRule INSTANCE = + FilterTableFunctionTransposeRule.Config.DEFAULT + .withOperandSupplier(b0 -> + b0.operand(HiveFilter.class).oneInput( + b1 -> b1.operand(HiveTableFunctionScan.class).anyInputs())) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); - if (CollectionUtils.isEmpty(columnMappings)) { - return false; - } - - for (Integer inputRef : inputRefs) { - if (!tfs.containsInputRefMapping(inputRef)) { - return false; - } - } - return true; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java index 35fb3613c7ea..3bba37304c2b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java @@ -19,22 +19,11 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.hep.HepRelVertex; -import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Correlate; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.rules.PruneEmptyRules; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.tools.RelBuilder; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAntiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -44,11 +33,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; -import java.util.Collections; -import java.util.List; - -import static com.google.common.collect.Iterables.concat; - /** * This class provides access to Calcite's {@link PruneEmptyRules}. * The instances of the rules use {@link org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelBuilder}. @@ -81,41 +65,6 @@ private static RelOptRule getJoinLeftInstance(Class clazz .toRule(); } - /** - * Improved version of Calcite's {@link PruneEmptyRules.JoinLeftEmptyRuleConfig}. - * In case of right outer join if the left branch is empty the join operator can be removed - * and take the right branch only. - * - * select * from (select * from emp where 1=0) right join dept - * to - * select null as emp.col0 ... null as emp.coln, dept.* from dept - */ - private static final class JoinLeftEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5294_FIXED) { - throw new IllegalStateException( - "Class JoinLeftEmptyRuleConfig is redundant after fix is merged into Calcite"); - } - - final Join join = call.rel(0); - final RelNode right = call.rel(2); - final RelBuilder relBuilder = call.builder(); - if (join.getJoinType().generatesNullsOnLeft()) { - // If "emp" is empty, "select * from emp right join dept" will have - // the same number of rows as "dept", and null values for the - // columns from "emp". The left side of the join can be removed. - call.transformTo(padWithNulls(relBuilder, right, join.getRowType(), true)); - return; - } - call.transformTo(relBuilder.push(join).empty().build()); - } - }; - } - } - public static final RelOptRule JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveJoin.class); public static final RelOptRule ANTI_JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveAntiJoin.class); public static final RelOptRule SEMI_JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveSemiJoin.class); @@ -131,60 +80,6 @@ private static RelOptRule getJoinRightInstance(Class claz .toRule(); } - /** - * Improved version of Calcite's {@link PruneEmptyRules.JoinRightEmptyRuleConfig}. - * In case of left outer join if the right branch is empty the join operator can be removed - * and take the left branch only. - * - * select * from emp right join (select * from dept where 1=0) - * to - * select emp.*, null as dept.col0 ... null as dept.coln from emp - */ - private static final class JoinRightEmptyRuleConfig extends HiveRuleConfig implements - PruneEmptyRules.JoinRightEmptyRuleConfig { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5294_FIXED) { - throw new IllegalStateException( - "Class JoinRightEmptyRuleConfig is redundant after fix is merged into Calcite"); - } - - final Join join = call.rel(0); - final RelNode left = call.rel(1); - final RelBuilder relBuilder = call.builder(); - if (join.getJoinType().generatesNullsOnRight()) { - // If "dept" is empty, "select * from emp left join dept" will have - // the same number of rows as "emp", and null values for the - // columns from "dept". The right side of the join can be removed. - call.transformTo(padWithNulls(relBuilder, left, join.getRowType(), false)); - return; - } - if (join.getJoinType() == JoinRelType.ANTI) { - // In case of anti join: Join(X, Empty, ANTI) becomes X - call.transformTo(join.getLeft()); - return; - } - call.transformTo(relBuilder.push(join).empty().build()); - } - }; - } - } - - private static RelNode padWithNulls(RelBuilder builder, RelNode input, RelDataType resultType, - boolean leftPadding) { - int padding = resultType.getFieldCount() - input.getRowType().getFieldCount(); - List nullLiterals = Collections.nCopies(padding, builder.literal(null)); - builder.push(input); - if (leftPadding) { - builder.project(concat(nullLiterals, builder.fields())); - } else { - builder.project(concat(builder.fields(), nullLiterals)); - } - return builder.convert(resultType, true).build(); - } - public static final RelOptRule CORRELATE_RIGHT_INSTANCE = new CorrelateRightEmptyRuleConfig() .withOperandSupplier(b0 -> b0.operand(Correlate.class).inputs( @@ -200,57 +95,6 @@ private static RelNode padWithNulls(RelBuilder builder, RelNode input, RelDataTy .withDescription("PruneEmptyCorrelate(left)") .toRule(); - /** Configuration for rule that prunes a correlate if left input is empty. */ - public static class CorrelateLeftEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5669_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - final Correlate corr = call.rel(0); - call.transformTo(call.builder().push(corr).empty().build()); - } - }; - } - } - - /** Configuration for rule that prunes a correlate if right input is empty. */ - public static class CorrelateRightEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5669_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - final Correlate corr = call.rel(0); - final RelNode left = call.rel(1); - final RelBuilder b = call.builder(); - final RelNode newRel; - switch (corr.getJoinType()) { - case LEFT: - newRel = padWithNulls(b, left, corr.getRowType(), false); - break; - case INNER: - case SEMI: - newRel = b.push(corr).empty().build(); - break; - case ANTI: - newRel = left; - break; - default: - throw new IllegalStateException("Correlate does not support " + corr.getJoinType()); - } - call.transformTo(newRel); - } - }; - } - } - public static final RelOptRule SORT_INSTANCE = new RemoveEmptySingleRuleConfig() .withOperandFor(HiveSortLimit.class, singleRel -> true) @@ -270,7 +114,7 @@ public void onMatch(RelOptRuleCall call) { .toRule(); public static final RelOptRule UNION_INSTANCE = - new HiveUnionEmptyPruneRuleConfig() + new UnionEmptyPruneRuleConfig() .withOperandSupplier(b0 -> b0.operand(HiveUnion.class).unorderedInputs(b1 -> b1.operand(Values.class) @@ -278,77 +122,32 @@ public void onMatch(RelOptRuleCall call) { .withDescription("HivePruneEmptyUnionBranch") .toRule(); - /** - * Copy of {@link PruneEmptyRules.UnionEmptyPruneRuleConfig} but this version expects {@link Union}. - */ - private static final class HiveUnionEmptyPruneRuleConfig extends HiveRuleConfig implements PruneEmptyRules.PruneEmptyRule.Config { - @Override - public PruneEmptyRules.PruneEmptyRule toRule() { - return new PruneEmptyRules.PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5293_FIXED) { - throw new IllegalStateException( - "Class HiveUnionEmptyPruneRuleConfig is redundant after fix is merged into Calcite"); - } + private static final class RemoveEmptySingleRuleConfig extends HiveRuleConfig + implements RemoveEmptySingleRule.RemoveEmptySingleRuleConfig { + } - final Union union = call.rel(0); - final List inputs = union.getInputs(); - assert inputs != null; - final RelBuilder builder = call.builder(); - int nonEmptyInputs = 0; - for (RelNode input : inputs) { - if (!isEmpty(input)) { - builder.push(input); - nonEmptyInputs++; - } - } - assert nonEmptyInputs < inputs.size() - : "planner promised us at least one Empty child: " - + RelOptUtil.toString(union); - if (nonEmptyInputs == 0) { - builder.push(union).empty(); - } else { - builder.union(union.all, nonEmptyInputs); - builder.convert(union.getRowType(), true); - } - call.transformTo(builder.build()); - } - }; - } + private static final class SortFetchZeroRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.SortFetchZeroRuleConfig { } - private static boolean isEmpty(RelNode node) { - if (Bug.CALCITE_5293_FIXED) { - throw new IllegalStateException( - "Method HiveRemoveEmptySingleRules.isEmpty is redundant after fix is merged into Calcite"); - } + private static final class UnionEmptyPruneRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.UnionEmptyPruneRuleConfig { + } - if (node instanceof Values) { - return ((Values) node).getTuples().isEmpty(); - } - if (node instanceof HepRelVertex) { - return isEmpty(((HepRelVertex) node).getCurrentRel()); - } - // Note: relation input might be a RelSubset, so we just iterate over the relations - // in order to check if the subset is equivalent to an empty relation. - if (!(node instanceof RelSubset)) { - return false; - } - RelSubset subset = (RelSubset) node; - for (RelNode rel : subset.getRels()) { - if (isEmpty(rel)) { - return true; - } - } - return false; + private static final class JoinLeftEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.JoinLeftEmptyRuleConfig { } - private static final class RemoveEmptySingleRuleConfig extends HiveRuleConfig - implements RemoveEmptySingleRule.RemoveEmptySingleRuleConfig { + private static final class JoinRightEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.JoinRightEmptyRuleConfig { } - private static final class SortFetchZeroRuleConfig extends HiveRuleConfig - implements PruneEmptyRules.SortFetchZeroRuleConfig { + private static final class CorrelateLeftEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.CorrelateLeftEmptyRuleConfig { + } + + private static final class CorrelateRightEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.CorrelateRightEmptyRuleConfig { } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java index 73ee02a9b355..99affbf4624d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java @@ -18,143 +18,24 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Union; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Pair; -import org.apache.calcite.util.mapping.Mappings; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; +import org.apache.calcite.rel.rules.UnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.ImmutableList; /** * Planner rule that pulls up constants through a Union operator. */ -public class HiveUnionPullUpConstantsRule extends RelOptRule { - - protected static final Logger LOG = LoggerFactory.getLogger(HiveUnionPullUpConstantsRule.class); - +public class HiveUnionPullUpConstantsRule { - public static final HiveUnionPullUpConstantsRule INSTANCE = - new HiveUnionPullUpConstantsRule(HiveUnion.class, - HiveRelFactories.HIVE_BUILDER); - - private HiveUnionPullUpConstantsRule( - Class unionClass, - RelBuilderFactory relBuilderFactory) { - super(operand(unionClass, any()), - relBuilderFactory, null); + private HiveUnionPullUpConstantsRule() { + throw new IllegalStateException("Instantiation not allowed"); } - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5337_FIXED) { - throw new IllegalStateException("Class redundant when the fix for CALCITE-5337 is merged into Calcite"); - } - - final Union union = call.rel(0); - - final int count = union.getRowType().getFieldCount(); - if (count == 1) { - // No room for optimization since we cannot create an empty - // Project operator. - return; - } - - final RexBuilder rexBuilder = union.getCluster().getRexBuilder(); - final RelMetadataQuery mq = call.getMetadataQuery(); - final RelOptPredicateList predicates = mq.getPulledUpPredicates(union); - if (predicates == null) { - return; - } - - Map constants = new HashMap<>(); - for (int i = 0; i < count ; i++) { - RexNode expr = rexBuilder.makeInputRef(union, i); - if (predicates.constantMap.containsKey(expr)) { - constants.put(expr, predicates.constantMap.get(expr)); - } - } - - // None of the expressions are constant. Nothing to do. - if (constants.isEmpty()) { - return; - } - - // Create expressions for Project operators before and after the Union - List fields = union.getRowType().getFieldList(); - List topChildExprs = new ArrayList<>(); - List topChildExprsFields = new ArrayList<>(); - List refs = new ArrayList<>(); - ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder(); - for (int i = 0; i < count ; i++) { - RexNode expr = rexBuilder.makeInputRef(union, i); - RelDataTypeField field = fields.get(i); - if (constants.containsKey(expr)) { - if (constants.get(expr).getType().equals(field.getType())) { - topChildExprs.add(constants.get(expr)); - } else { - topChildExprs.add(rexBuilder.makeCast(field.getType(), constants.get(expr), true)); - } - topChildExprsFields.add(field.getName()); - } else { - topChildExprs.add(expr); - topChildExprsFields.add(field.getName()); - refs.add(expr); - refsIndexBuilder.set(i); - } - } - ImmutableBitSet refsIndex = refsIndexBuilder.build(); - - // Update top Project positions - final Mappings.TargetMapping mapping = - RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse(); - topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs)); - - // Create new Project-Union-Project sequences - final RelBuilder relBuilder = call.builder(); - for (int i = 0; i < union.getInputs().size() ; i++) { - RelNode input = union.getInput(i); - List> newChildExprs = new ArrayList<>(); - for (int j = 0; j < refsIndex.cardinality(); j++) { - int pos = refsIndex.nth(j); - newChildExprs.add(Pair.of(rexBuilder.makeInputRef(input, pos), - input.getRowType().getFieldList().get(pos).getName())); - } - if (newChildExprs.isEmpty()) { - // At least a single item in project is required. - newChildExprs.add(Pair.of(topChildExprs.get(0), topChildExprsFields.get(0))); - } - // Add the input with project on top - relBuilder.push(input); - relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs)); - } - relBuilder.union(union.all, union.getInputs().size()); - // Create top Project fixing nullability of fields - relBuilder.project(topChildExprs, topChildExprsFields); - relBuilder.convert(union.getRowType(), false); - - call.transformTo(relBuilder.build()); - } + public static final RelOptRule INSTANCE = + UnionPullUpConstantsRule.Config.DEFAULT + .withOperandFor(HiveUnion.class) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); } From 4d5100e1897161af6c6dfee7be231f40bf41e681 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 25 Aug 2026 15:15:26 +0100 Subject: [PATCH 4/6] Minor changes to fix sonar issues --- .../calcite/rules/HiveFilterTableFunctionTransposeRule.java | 4 ++-- .../optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java index b4f3f01df96d..6fb73ea2b3ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java @@ -25,9 +25,9 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; /** - * Rule to transpose Filter and TableFunctionScan RelNodes + * Rule to transpose Filter and TableFunctionScan RelNodes. */ -public class HiveFilterTableFunctionTransposeRule { +public final class HiveFilterTableFunctionTransposeRule { private HiveFilterTableFunctionTransposeRule() { throw new IllegalStateException("Instantiation not allowed"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java index 99affbf4624d..16018ffc2f40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java @@ -26,7 +26,7 @@ /** * Planner rule that pulls up constants through a Union operator. */ -public class HiveUnionPullUpConstantsRule { +public final class HiveUnionPullUpConstantsRule { private HiveUnionPullUpConstantsRule() { throw new IllegalStateException("Instantiation not allowed"); From 8ef424463e7c90090d23338f1fb2144711162fad Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 25 Aug 2026 19:02:15 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Put=20back=20HiveFilterTableFunctionTranspo?= =?UTF-8?q?seRule=20code,=20since=20it=20does=20not=20seem=20identical=20t?= =?UTF-8?q?o=20Calcite's=20version,=20the=20latter=20causes=20issues=20wit?= =?UTF-8?q?h=20certain=20tests:=20-=20Testing=20/=20split-14=20/=20PostPro?= =?UTF-8?q?cess=20/=20testCliDriver[lvj=5Fmapjoin]=20=E2=80=93=20org.apach?= =?UTF-8?q?e.hadoop.hive.cli.split13.TestMiniLlapLocalCliDriver=20-=20Test?= =?UTF-8?q?ing=20/=20split-05=20/=20PostProcess=20/=20testCliDriver[latera?= =?UTF-8?q?l=5Fview=5Fcbo]=20=E2=80=93=20org.apache.hadoop.hive.cli.split1?= =?UTF-8?q?2.TestMiniLlapLocalCliDriver=20-=20Testing=20/=20split-05=20/?= =?UTF-8?q?=20PostProcess=20/=20testCliDriver[lateral=5Fview=5Fppd]=20?= =?UTF-8?q?=E2=80=93=20org.apache.hadoop.hive.cli.split12.TestMiniLlapLoca?= =?UTF-8?q?lCliDriver=20-=20Testing=20/=20split-10=20/=20PostProcess=20/?= =?UTF-8?q?=20testCliDriver[tez=5Funion=5Fudtf]=20=E2=80=93=20org.apache.h?= =?UTF-8?q?adoop.hive.cli.TestMiniTezCliDriver=20-=20Testing=20/=20split-1?= =?UTF-8?q?9=20/=20PostProcess=20/=20testCliDriver[union26]=20=E2=80=93=20?= =?UTF-8?q?org.apache.hadoop.hive.cli.split27.TestMiniLlapLocalCliDriver?= =?UTF-8?q?=20-=20Testing=20/=20split-04=20/=20PostProcess=20/=20testCliDr?= =?UTF-8?q?iver[nonmr=5Ffetch]=20=E2=80=93=20org.apache.hadoop.hive.cli.sp?= =?UTF-8?q?lit7.TestCliDriver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../HiveFilterTableFunctionTransposeRule.java | 130 ++++++++++++++++-- 1 file changed, 118 insertions(+), 12 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java index 6fb73ea2b3ec..74be18e6e937 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java @@ -19,26 +19,132 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.rel.rules.FilterTableFunctionTransposeRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.metadata.RelColumnMapping; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + /** * Rule to transpose Filter and TableFunctionScan RelNodes. + * This rule's code has diverged from Calcite's FilterTableFunctionTransposeRule. */ -public final class HiveFilterTableFunctionTransposeRule { +// TODO could this be reconciled with Calcite's rule? +public class HiveFilterTableFunctionTransposeRule extends RelOptRule { + + public static final HiveFilterTableFunctionTransposeRule INSTANCE = + new HiveFilterTableFunctionTransposeRule(HiveRelFactories.HIVE_BUILDER); + + public HiveFilterTableFunctionTransposeRule(RelBuilderFactory relBuilderFactory) { + super(operand(HiveFilter.class, operand(HiveTableFunctionScan.class, any())), + relBuilderFactory, null); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final Filter filterRel = call.rel(0); + final HiveTableFunctionScan tfs = call.rel(1); + + RexNode condition = filterRel.getCondition(); + if (!HiveCalciteUtil.isDeterministic(condition)) { + return false; + } + + // The TableFunctionScan is always created such that all the input RelNode + // fields are present in its RelNode. If a Filter has an InputRef that is + // greater then the number of the RelNode below the TableFunctionScan, that + // means it was a field created by the TableFunctionScan and thus the Filter + // cannot be pushed through. + // + // We check for each individual conjunction (breaking it up by top level 'and' + // conditions). + for (RexNode ce : RelOptUtil.conjunctions(filterRel.getCondition())) { + if (canBePushed(HiveCalciteUtil.getInputRefs(ce), tfs)) { + return true; + } + } + return false; + } + + public void onMatch(RelOptRuleCall call) { + final Filter filter = call.rel(0); + final HiveTableFunctionScan tfs = call.rel(1); + final RelBuilder builder = call.builder(); - private HiveFilterTableFunctionTransposeRule() { - throw new IllegalStateException("Instantiation not allowed"); + final List newPartKeyFilterConditions = new ArrayList<>(); + final List unpushedFilterConditions = new ArrayList<>(); + + // Check for each individual 'and' condition so that we can push partial + // expressions through. + for (RexNode ce : RelOptUtil.conjunctions(filter.getCondition())) { + // We can only push if all the InputRef pointers are referencing the + // input RelNode to the TableFunctionScan + if (canBePushed(HiveCalciteUtil.getInputRefs(ce), tfs)) { + newPartKeyFilterConditions.add(ce); + } else { + unpushedFilterConditions.add(ce); + } + } + + // The "matches" check should guarantee there's something to push. + final RexNode filterCondToPushBelowProj = RexUtil.composeConjunction( + filter.getCluster().getRexBuilder(), newPartKeyFilterConditions, true); + + builder.push(tfs.getInput(0)).filter(filterCondToPushBelowProj); + + // If there are conditions that cannot be pushed through, generate the RexNode + final RexNode unpushedFilCondAboveProj = unpushedFilterConditions.isEmpty() + ? null + : RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), + unpushedFilterConditions, true); + + // Generate the new TableFunctionScanNode with the Filter InputRel + final RelNode tableFunctionScanNode = tfs.copy(tfs.getTraitSet(), + ImmutableList.of(builder.build()), tfs.getCall(), tfs.getElementType(), + tfs.getRowType(), tfs.getColumnMappings()); + + builder.clear(); + builder.push(tableFunctionScanNode); + + if (unpushedFilCondAboveProj != null) { + builder.filter(unpushedFilCondAboveProj); + } + + call.transformTo(builder.build()); } - public static final RelOptRule INSTANCE = - FilterTableFunctionTransposeRule.Config.DEFAULT - .withOperandSupplier(b0 -> - b0.operand(HiveFilter.class).oneInput( - b1 -> b1.operand(HiveTableFunctionScan.class).anyInputs())) - .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) - .toRule(); + // If any of the inputRefs are references to a field that is not mapped into the inputRelNode, + // the condition cannot be pushed. + private boolean canBePushed(Set inputRefs, HiveTableFunctionScan tfs) { + Set columnMappings = tfs.getColumnMappings(); + if (inputRefs.isEmpty()) { + return true; + } -} + if (CollectionUtils.isEmpty(columnMappings)) { + return false; + } + + for (Integer inputRef : inputRefs) { + if (!tfs.containsInputRefMapping(inputRef)) { + return false; + } + } + return true; + } +} \ No newline at end of file From 5dedbc1206059f8a54ffa96d10016bcb6293d078 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 26 Aug 2026 08:34:54 +0100 Subject: [PATCH 6/6] More cleanup --- .../hadoop/hive/ql/optimizer/calcite/Bug.java | 15 --- .../ql/optimizer/calcite/HiveRelOptUtil.java | 14 --- .../ql/optimizer/calcite/HiveTypeFactory.java | 27 ------ .../HiveFilterTableFunctionTransposeRule.java | 2 +- .../calcite/stats/HiveRelMdUniqueKeys.java | 96 ------------------- 5 files changed, 1 insertion(+), 153 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java index da7fbfd78aec..c53fa4a95bec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -42,25 +42,10 @@ public final class Bug { */ public static final boolean CALCITE_4166_FIXED = false; - /** - * Whether CALCITE-6513 is fixed. - */ - public static final boolean CALCITE_6513_FIXED = false; - - /** - * Whether CALCITE-6704 is fixed. - */ - public static final boolean CALCITE_6704_FIXED = false; - /** * Whether CALCITE-6737 is fixed. */ public static final boolean CALCITE_6737_FIXED = false; - - /** - * Whether CALCITE-6954 is fixed. - */ - public static final boolean CALCITE_6954_FIXED = false; /** * Whether CALCITE-6832 is fixed. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index 1d21c8a2c57e..687d87f8af24 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -728,20 +728,6 @@ public static PKFKJoinInfo extractPKFKJoin( return cannotExtract; } - public static RexNode pushPastProjectUnlessBloat(RexNode node, Project project, int bloat) { - if (Bug.CALCITE_6513_FIXED) { - throw new IllegalStateException("Method is redundant when the fix for CALCITE-6513 is merged into Calcite. " + - "Use RelOptUtil.pushPastProjectUnlessBloat"); - } - - List newConditions = - RelOptUtil.pushPastProjectUnlessBloat(Collections.singletonList(node), project, bloat); - if (newConditions == null || newConditions.size() != 1) { - return null; - } - return newConditions.get(0); - } - public static class PKFKJoinInfo { public final boolean isPkFkJoin; public final Pair pkFkJoinColumns; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java index 06995d6625ad..4e32dcb8dacc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java @@ -19,40 +19,13 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ConversionUtil; -import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; -import java.util.List; public class HiveTypeFactory extends JavaTypeFactoryImpl { public HiveTypeFactory() { super(new HiveTypeSystemImpl()); - if (Bug.CALCITE_6954_FIXED) { - throw new IllegalStateException("Class redundant once fix is merged"); - } - } - - @Override - protected @Nullable RelDataType leastRestrictiveArrayMultisetType(final List types, - final SqlTypeName sqlTypeName) { - RelDataType type = super.leastRestrictiveArrayMultisetType(types, sqlTypeName); - if (type != null) { - return canonize(type); - } - return null; - } - - @Override - protected @Nullable RelDataType leastRestrictiveMapType(final List types, - final SqlTypeName sqlTypeName) { - RelDataType type = super.leastRestrictiveMapType(types, sqlTypeName); - if (type != null) { - return canonize(type); - } - return null; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java index 74be18e6e937..3b1978abfbdc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java @@ -147,4 +147,4 @@ private boolean canBePushed(Set inputRefs, HiveTableFunctionScan tfs) { } return true; } -} \ No newline at end of file +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 6e0430f4019e..9a9f48030a59 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -22,124 +22,28 @@ import java.util.List; import java.util.Set; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.rel.SingleRel; -import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.MetadataDef; import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Util; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; - public class HiveRelMdUniqueKeys implements MetadataHandler { public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( BuiltInMethod.UNIQUE_KEYS.method, new HiveRelMdUniqueKeys()); - /** - * A limit about the number of unique keys returned by the handler. - * The limit must be in the range [0, Integer.MAX_VALUE]. - */ - private final int limit; - - private HiveRelMdUniqueKeys() { - if (Bug.CALCITE_6704_FIXED) { - throw new IllegalStateException("Remove constructor and limit once we upgrade to version with CALCITE-6704"); - } - this.limit = 1000; - } @Override public MetadataDef getDef() { return BuiltInMetadata.UniqueKeys.DEF; } - public Set getUniqueKeys(Project rel, RelMetadataQuery mq, - boolean ignoreNulls) { - return getProjectUniqueKeys(rel, mq, ignoreNulls, rel.getProjects()); - } - - private Set getProjectUniqueKeys(SingleRel rel, RelMetadataQuery mq, - boolean ignoreNulls, List projExprs) { - if (Bug.CALCITE_6704_FIXED) { - throw new IllegalStateException("Method is redundant once we upgrade to version with CALCITE-6704"); - - } - // LogicalProject maps a set of rows to a different set; - // Without knowledge of the mapping function(whether it - // preserves uniqueness), it is only safe to derive uniqueness - // info from the child of a project when the mapping is f(a) => a. - // - // Further more, the unique bitset coming from the child needs - // to be mapped to match the output of the project. - - // Single input can be mapped to multiple outputs - ImmutableMultimap.Builder inToOutPosBuilder = ImmutableMultimap.builder(); - ImmutableBitSet.Builder mappedInColumnsBuilder = ImmutableBitSet.builder(); - - // Build an input to output position map. - for (int i = 0; i < projExprs.size(); i++) { - RexNode projExpr = projExprs.get(i); - if (projExpr instanceof RexInputRef) { - int inputIndex = ((RexInputRef) projExpr).getIndex(); - inToOutPosBuilder.put(inputIndex, i); - mappedInColumnsBuilder.set(inputIndex); - } - } - ImmutableBitSet inColumnsUsed = mappedInColumnsBuilder.build(); - - if (inColumnsUsed.isEmpty()) { - // if there's no RexInputRef in the projected expressions - // return empty set. - return ImmutableSet.of(); - } - - Set childUniqueKeySet = - mq.getUniqueKeys(rel.getInput(), ignoreNulls); - - if (childUniqueKeySet == null) { - return ImmutableSet.of(); - } - - Multimap mapInToOutPos = inToOutPosBuilder.build(); - - Set resultBuilder = new HashSet<>(); - // Now add to the projUniqueKeySet the child keys that are fully - // projected. - outerLoop: - for (ImmutableBitSet colMask : childUniqueKeySet) { - if (!inColumnsUsed.contains(colMask)) { - // colMask contains a column that is not projected as RexInput => the key is not unique - continue; - } - // colMask is mapped to output project, however, the column can be mapped more than once: - // select key1, key1, val1, val2, key2 from ... - // the resulting unique keys would be {{0},{4}}, {{1},{4}} - - Iterable> product = Linq4j.product(Util.transform(colMask.toList(), mapInToOutPos::get)); - for (List passKey : product) { - if (resultBuilder.size() == limit) { - break outerLoop; - } - resultBuilder.add(ImmutableBitSet.of(passKey)); - } - } - return resultBuilder; - } - public Set getUniqueKeys(HiveTableScan rel, RelMetadataQuery mq, boolean ignoreNulls) { RelOptHiveTable tbl = (RelOptHiveTable) rel.getTable();