diff --git a/data/conf/hive-log4j2.properties b/data/conf/hive-log4j2.properties index 5eb2480a2a6e..bbcec098dd8c 100644 --- a/data/conf/hive-log4j2.properties +++ b/data/conf/hive-log4j2.properties @@ -114,7 +114,7 @@ logger.CalcitePlanner.name = org.apache.calcite.plan.RelOptPlanner logger.CalcitePlanner.level = INFO # Change logger level to DEBUG, to see all application of CBO rules and the operators they produce -logger.CBORuleLogger.name = org.apache.hadoop.hive.ql.optimizer.calcite.RuleEventLogger +logger.CBORuleLogger.name = org.apache.calcite.plan.RuleEventLogger logger.CBORuleLogger.level = OFF logger.CBORuleLogger.filter.marker.type = MarkerFilter logger.CBORuleLogger.filter.marker.marker = FULL_PLAN diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java index fbd48a721e5a..c53fa4a95bec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java @@ -42,78 +42,10 @@ public final class Bug { */ public static final boolean CALCITE_4166_FIXED = false; - /** - * Whether issue - * CALCITE-4499 is fixed. - */ - public static final boolean CALCITE_4499_FIXED = false; - - /** - * Whether issue - * CALCITE-4200 is fixed. - */ - public static final boolean CALCITE_4200_FIXED = false; - - /** - * Whether issue - * CALCITE-4513 is fixed. - */ - public static final boolean CALCITE_4513_FIXED=false; - - /** - * Whether CALCITE-4574 is fixed. - */ - public static final boolean CALCITE_4574_FIXED = false; - - /** - * Whether CALCITE-4704 is fixed. - */ - public static final boolean CALCITE_4704_FIXED = false; - - /** - * Whether CALCITE-5293 is fixed. - */ - public static final boolean CALCITE_5293_FIXED = false; - - /** - * Whether CALCITE-5294 is fixed. - */ - public static final boolean CALCITE_5294_FIXED = false; - - /** - * Whether CALCITE-5337 is fixed. - */ - public static final boolean CALCITE_5337_FIXED = false; - - /** - * Whether CALCITE-5669 is fixed. - */ - public static final boolean CALCITE_5669_FIXED = false; - - /** - * Whether CALCITE-5985 is fixed. - */ - public static final boolean CALCITE_5985_FIXED = false; - - /** - * Whether CALCITE-6513 is fixed. - */ - public static final boolean CALCITE_6513_FIXED = false; - - /** - * Whether CALCITE-6704 is fixed. - */ - public static final boolean CALCITE_6704_FIXED = false; - /** * Whether CALCITE-6737 is fixed. */ public static final boolean CALCITE_6737_FIXED = false; - - /** - * Whether CALCITE-6954 is fixed. - */ - public static final boolean CALCITE_6954_FIXED = false; /** * Whether CALCITE-6832 is fixed. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java index d76c545b3aba..068e56acdfb4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java @@ -18,23 +18,13 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite; -import com.google.common.collect.Iterables; import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptSchema; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.server.CalciteServerStatement; @@ -44,14 +34,8 @@ import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Litmus; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - /** * Builder for relational expressions in Hive. * @@ -90,7 +74,7 @@ public static RelBuilderFactory proto(final Context context) { return new RelBuilderFactory() { @Override public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) { - Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED) + Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(false) // TODO clarify CALCITE_4513 .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema); } @@ -133,88 +117,4 @@ public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) { } return SqlStdOperatorTable.FLOOR; } - - /** Creates a {@link Join} with correlating variables. */ - @Override - public RelBuilder join(JoinRelType joinType, RexNode condition, - Set variablesSet) { - if (Bug.CALCITE_4574_FIXED) { - throw new IllegalStateException("Method overriding should be removed once CALCITE-4574 is fixed"); - } - RelNode right = this.peek(0); - RelNode left = this.peek(1); - final boolean correlate = variablesSet.size() == 1; - RexNode postCondition = literal(true); - if (correlate) { - final CorrelationId id = Iterables.getOnlyElement(variablesSet); - if (!RelOptUtil.notContainsCorrelation(left, id, Litmus.IGNORE)) { - throw new IllegalArgumentException("variable " + id - + " must not be used by left input to correlation"); - } - // Correlate does not have an ON clause. - switch (joinType) { - case LEFT: - case SEMI: - case ANTI: - // For a LEFT/SEMI/ANTI, predicate must be evaluated first. - filter(condition.accept(new Shifter(left, id, right))); - right = this.peek(0); - break; - case INNER: - // For INNER, we can defer. - postCondition = condition; - break; - default: - throw new IllegalArgumentException("Correlated " + joinType + " join is not supported"); - } - final ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, right); - List leftFields = this.fields(2, 0); - List requiredFields = new ArrayList<>(); - for (int i = 0; i < leftFields.size(); i++) { - if (requiredColumns.get(i)) { - requiredFields.add(leftFields.get(i)); - } - } - correlate(joinType, id, requiredFields); - filter(postCondition); - } else { - // When there is no correlation use the default logic which works OK for now - // Cannot copy-paste the respective code here cause we don't have access to stack, - // Frame etc. and we might lose existing aliases in the builder - assert variablesSet.isEmpty(); - super.join(joinType,condition, variablesSet); - } - return this; - } - - /** Shuttle that shifts a predicate's inputs to the left, replacing early - * ones with references to a - * {@link RexCorrelVariable}. */ - private class Shifter extends RexShuttle { - private final RelNode left; - private final CorrelationId id; - private final RelNode right; - - Shifter(RelNode left, CorrelationId id, RelNode right) { - this.left = left; - this.id = id; - this.right = right; - if (Bug.CALCITE_4574_FIXED) { - throw new IllegalStateException("Class should be redundant once CALCITE-4574 is fixed"); - } - } - - public RexNode visitInputRef(RexInputRef inputRef) { - final RelDataType leftRowType = left.getRowType(); - final RexBuilder rexBuilder = getRexBuilder(); - final int leftCount = leftRowType.getFieldCount(); - if (inputRef.getIndex() < leftCount) { - final RexNode v = rexBuilder.makeCorrel(leftRowType, id); - return rexBuilder.makeFieldAccess(v, inputRef.getIndex()); - } else { - return rexBuilder.makeInputRef(right, inputRef.getIndex() - leftCount); - } - } - } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index 1d21c8a2c57e..687d87f8af24 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -728,20 +728,6 @@ public static PKFKJoinInfo extractPKFKJoin( return cannotExtract; } - public static RexNode pushPastProjectUnlessBloat(RexNode node, Project project, int bloat) { - if (Bug.CALCITE_6513_FIXED) { - throw new IllegalStateException("Method is redundant when the fix for CALCITE-6513 is merged into Calcite. " + - "Use RelOptUtil.pushPastProjectUnlessBloat"); - } - - List newConditions = - RelOptUtil.pushPastProjectUnlessBloat(Collections.singletonList(node), project, bloat); - if (newConditions == null || newConditions.size() != 1) { - return null; - } - return newConditions.get(0); - } - public static class PKFKJoinInfo { public final boolean isPkFkJoin; public final Pair pkFkJoinColumns; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java index 06995d6625ad..4e32dcb8dacc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeFactory.java @@ -19,40 +19,13 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ConversionUtil; -import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; -import java.util.List; public class HiveTypeFactory extends JavaTypeFactoryImpl { public HiveTypeFactory() { super(new HiveTypeSystemImpl()); - if (Bug.CALCITE_6954_FIXED) { - throw new IllegalStateException("Class redundant once fix is merged"); - } - } - - @Override - protected @Nullable RelDataType leastRestrictiveArrayMultisetType(final List types, - final SqlTypeName sqlTypeName) { - RelDataType type = super.leastRestrictiveArrayMultisetType(types, sqlTypeName); - if (type != null) { - return canonize(type); - } - return null; - } - - @Override - protected @Nullable RelDataType leastRestrictiveMapType(final List types, - final SqlTypeName sqlTypeName) { - RelDataType type = super.leastRestrictiveMapType(types, sqlTypeName); - if (type != null) { - return canonize(type); - } - return null; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java deleted file mode 100644 index 248fd741abed..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RuleEventLogger.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite; - -import org.apache.calcite.plan.RelOptListener; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.Marker; -import org.slf4j.MarkerFactory; - -import java.util.Arrays; -import java.util.stream.Collectors; - -/** - * Listener for logging useful debugging information on certain rule events. - */ -public class RuleEventLogger implements RelOptListener { - private static final Logger LOG = LoggerFactory.getLogger(RuleEventLogger.class.getName()); - private static final Marker FULL = MarkerFactory.getMarker("FULL_PLAN"); - - public RuleEventLogger() { - if (Bug.CALCITE_4704_FIXED) { - throw new IllegalStateException("Class redundant after fix is merged into Calcite"); - } - } - - @Override - public void relEquivalenceFound(final RelEquivalenceEvent event) { - - } - - @Override - public void ruleAttempted(final RuleAttemptedEvent event) { - if (event.isBefore() && LOG.isDebugEnabled()) { - RelOptRuleCall call = event.getRuleCall(); - String ruleArgs = Arrays.stream(call.rels).map(rel -> "rel#" + rel.getId() + ":" + rel.getRelTypeName()) - .collect(Collectors.joining(",")); - LOG.debug("call#{}: Apply rule [{}] to [{}]", call.id, call.getRule(), ruleArgs); - } - } - - @Override - public void ruleProductionSucceeded(RuleProductionEvent event) { - if (event.isBefore() && LOG.isDebugEnabled()) { - RelOptRuleCall call = event.getRuleCall(); - - Arrays.stream(call.rels).forEach(rel -> - LOG.debug(FULL, "call#{}: Full plan for rule input [rel#{}:{}]: {}", call.id, rel.getId(), - rel.getRelTypeName(), System.lineSeparator() + RelOptUtil.toString(rel))); - - RelNode newRel = event.getRel(); - String description = newRel == null ? "null" : "rel#" + newRel.getId() + ":" + newRel.getRelTypeName(); - LOG.debug("call#{}: Rule [{}] produced [{}]", call.id, call.getRule(), description); - if (newRel != null) { - LOG.debug(FULL, "call#{}: Full plan for [{}]:{}", call.id, description, - System.lineSeparator() + RelOptUtil.toString(newRel)); - } - } - } - - @Override - public void relDiscarded(final RelDiscardedEvent event) { - - } - - @Override - public void relChosen(final RelChosenEvent event) { - - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java index ac4ba209fc80..38ee55b3ae95 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java @@ -27,22 +27,17 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.RelOptUtil.InputFinder; -import org.apache.calcite.plan.RelOptUtil.RexInputConverter; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; @@ -209,7 +204,7 @@ protected void perform(RelOptRuleCall call, Filter filter, Join join) { // Try to push down above filters. These are typically where clause // filters. They can be pushed down if they are not on the NULL // generating side. - boolean filterPushed = classifyFilters( + boolean filterPushed = RelOptUtil.classifyFilters( join, aboveFilters, joinType == JoinRelType.INNER || joinType == JoinRelType.SEMI, @@ -250,7 +245,7 @@ protected void perform(RelOptRuleCall call, Filter filter, Join join) { // that is, the result set may be smaller. The right can not be pushed // into for the same reason. if (joinType != JoinRelType.ANTI - && classifyFilters( + && RelOptUtil.classifyFilters( join, joinFilters, false, @@ -341,150 +336,6 @@ private List getConjunctions(Filter filter) { } return conjunctions; } - - /** - * Classifies filters according to where they should be processed. They - * either stay where they are, are pushed to the join (if they originated - * from above the join), or are pushed to one of the children. Filters that - * are pushed are added to list passed in as input parameters. - * - * @param joinRel join node - * @param filters filters to be classified - * @param pushInto whether filters can be pushed into the join - * @param pushLeft true if filters can be pushed to the left - * @param pushRight true if filters can be pushed to the right - * @param joinFilters list of filters to push to the join - * @param leftFilters list of filters to push to the left child - * @param rightFilters list of filters to push to the right child - * @return whether at least one filter was pushed - */ - private static boolean classifyFilters( - RelNode joinRel, - List filters, - boolean pushInto, - boolean pushLeft, - boolean pushRight, - List joinFilters, - List leftFilters, - List rightFilters) { - if (Bug.CALCITE_4499_FIXED) { - throw new AssertionError("Remove this method when [CALCITE-4499] " - + "has been fixed and use directly Calcite's RelOptUtil.classifyFilters."); - } - RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); - List joinFields = joinRel.getRowType().getFieldList(); - final int nSysFields = 0; // joinRel.getSystemFieldList().size(); - final List leftFields = - joinRel.getInputs().get(0).getRowType().getFieldList(); - final int nFieldsLeft = leftFields.size(); - final List rightFields = - joinRel.getInputs().get(1).getRowType().getFieldList(); - final int nFieldsRight = rightFields.size(); - final int nTotalFields = nFieldsLeft + nFieldsRight; - - // set the reference bitmaps for the left and right children - ImmutableBitSet leftBitmap = - ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft); - ImmutableBitSet rightBitmap = - ImmutableBitSet.range(nSysFields + nFieldsLeft, nTotalFields); - - final List filtersToRemove = new ArrayList<>(); - for (RexNode filter : filters) { - final InputFinder inputFinder = InputFinder.analyze(filter); - final ImmutableBitSet inputBits = inputFinder.build(); - - // REVIEW - are there any expressions that need special handling - // and therefore cannot be pushed? - - // filters can be pushed to the left child if the left child - // does not generate NULLs and the only columns referenced in - // the filter originate from the left child - if (pushLeft && leftBitmap.contains(inputBits)) { - // ignore filters that always evaluate to true - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the left now shift over by the number - // of system fields - final RexNode shiftedFilter = - shiftFilter( - nSysFields, - nSysFields + nFieldsLeft, - -nSysFields, - rexBuilder, - joinFields, - nTotalFields, - leftFields, - filter); - - leftFilters.add(shiftedFilter); - } - filtersToRemove.add(filter); - - // filters can be pushed to the right child if the right child - // does not generate NULLs and the only columns referenced in - // the filter originate from the right child - } else if (pushRight && rightBitmap.contains(inputBits)) { - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the right now shift over to the left; - // since we never push filters to a NULL generating - // child, the types of the source should match the dest - // so we don't need to explicitly pass the destination - // fields to RexInputConverter - final RexNode shiftedFilter = - shiftFilter( - nSysFields + nFieldsLeft, - nTotalFields, - -(nSysFields + nFieldsLeft), - rexBuilder, - joinFields, - nTotalFields, - rightFields, - filter); - rightFilters.add(shiftedFilter); - } - filtersToRemove.add(filter); - - } else { - // If the filter can't be pushed to either child, we may push them into the join - if (pushInto) { - if (!joinFilters.contains(filter)) { - joinFilters.add(filter); - } - filtersToRemove.add(filter); - } - } - } - - // Remove filters after the loop, to prevent concurrent modification. - if (!filtersToRemove.isEmpty()) { - filters.removeAll(filtersToRemove); - } - - // Did anything change? - return !filtersToRemove.isEmpty(); - } - - private static RexNode shiftFilter( - int start, - int end, - int offset, - RexBuilder rexBuilder, - List joinFields, - int nTotalFields, - List rightFields, - RexNode filter) { - int[] adjustments = new int[nTotalFields]; - for (int i = start; i < end; i++) { - adjustments[i] = offset; - } - return filter.accept( - new RexInputConverter( - rexBuilder, - joinFields, - rightFields, - adjustments)); - } } // End PushFilterPastJoinRule.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java index ee8562f34362..3b1978abfbdc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterTableFunctionTransposeRule.java @@ -24,20 +24,16 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.metadata.RelColumnMapping; -import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.commons.collections4.CollectionUtils; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -45,17 +41,14 @@ import java.util.Set; /** - * Rule to transpose Filter and TableFunctionScan RelNodes - * - * We cannot use Calcite's FilterTableFunctionTransposeRule because that rule - * uses LogicalFilter and LogicalTableFunctionScan. We should remove this - * class when CALCITE-5985 is fixed (and remove the CALCITE_5985_FIXED entry - * in Bug.java) + * Rule to transpose Filter and TableFunctionScan RelNodes. + * This rule's code has diverged from Calcite's FilterTableFunctionTransposeRule. */ +// TODO could this be reconciled with Calcite's rule? public class HiveFilterTableFunctionTransposeRule extends RelOptRule { public static final HiveFilterTableFunctionTransposeRule INSTANCE = - new HiveFilterTableFunctionTransposeRule(HiveRelFactories.HIVE_BUILDER); + new HiveFilterTableFunctionTransposeRule(HiveRelFactories.HIVE_BUILDER); public HiveFilterTableFunctionTransposeRule(RelBuilderFactory relBuilderFactory) { super(operand(HiveFilter.class, operand(HiveTableFunctionScan.class, any())), @@ -64,10 +57,6 @@ public HiveFilterTableFunctionTransposeRule(RelBuilderFactory relBuilderFactory) @Override public boolean matches(RelOptRuleCall call) { - if (Bug.CALCITE_5985_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - final Filter filterRel = call.rel(0); final HiveTableFunctionScan tfs = call.rel(1); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java index 35fb3613c7ea..3bba37304c2b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveEmptySingleRules.java @@ -19,22 +19,11 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.hep.HepRelVertex; -import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Correlate; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.rules.PruneEmptyRules; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.tools.RelBuilder; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAntiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -44,11 +33,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; -import java.util.Collections; -import java.util.List; - -import static com.google.common.collect.Iterables.concat; - /** * This class provides access to Calcite's {@link PruneEmptyRules}. * The instances of the rules use {@link org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelBuilder}. @@ -81,41 +65,6 @@ private static RelOptRule getJoinLeftInstance(Class clazz .toRule(); } - /** - * Improved version of Calcite's {@link PruneEmptyRules.JoinLeftEmptyRuleConfig}. - * In case of right outer join if the left branch is empty the join operator can be removed - * and take the right branch only. - * - * select * from (select * from emp where 1=0) right join dept - * to - * select null as emp.col0 ... null as emp.coln, dept.* from dept - */ - private static final class JoinLeftEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5294_FIXED) { - throw new IllegalStateException( - "Class JoinLeftEmptyRuleConfig is redundant after fix is merged into Calcite"); - } - - final Join join = call.rel(0); - final RelNode right = call.rel(2); - final RelBuilder relBuilder = call.builder(); - if (join.getJoinType().generatesNullsOnLeft()) { - // If "emp" is empty, "select * from emp right join dept" will have - // the same number of rows as "dept", and null values for the - // columns from "emp". The left side of the join can be removed. - call.transformTo(padWithNulls(relBuilder, right, join.getRowType(), true)); - return; - } - call.transformTo(relBuilder.push(join).empty().build()); - } - }; - } - } - public static final RelOptRule JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveJoin.class); public static final RelOptRule ANTI_JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveAntiJoin.class); public static final RelOptRule SEMI_JOIN_RIGHT_INSTANCE = getJoinRightInstance(HiveSemiJoin.class); @@ -131,60 +80,6 @@ private static RelOptRule getJoinRightInstance(Class claz .toRule(); } - /** - * Improved version of Calcite's {@link PruneEmptyRules.JoinRightEmptyRuleConfig}. - * In case of left outer join if the right branch is empty the join operator can be removed - * and take the left branch only. - * - * select * from emp right join (select * from dept where 1=0) - * to - * select emp.*, null as dept.col0 ... null as dept.coln from emp - */ - private static final class JoinRightEmptyRuleConfig extends HiveRuleConfig implements - PruneEmptyRules.JoinRightEmptyRuleConfig { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5294_FIXED) { - throw new IllegalStateException( - "Class JoinRightEmptyRuleConfig is redundant after fix is merged into Calcite"); - } - - final Join join = call.rel(0); - final RelNode left = call.rel(1); - final RelBuilder relBuilder = call.builder(); - if (join.getJoinType().generatesNullsOnRight()) { - // If "dept" is empty, "select * from emp left join dept" will have - // the same number of rows as "emp", and null values for the - // columns from "dept". The right side of the join can be removed. - call.transformTo(padWithNulls(relBuilder, left, join.getRowType(), false)); - return; - } - if (join.getJoinType() == JoinRelType.ANTI) { - // In case of anti join: Join(X, Empty, ANTI) becomes X - call.transformTo(join.getLeft()); - return; - } - call.transformTo(relBuilder.push(join).empty().build()); - } - }; - } - } - - private static RelNode padWithNulls(RelBuilder builder, RelNode input, RelDataType resultType, - boolean leftPadding) { - int padding = resultType.getFieldCount() - input.getRowType().getFieldCount(); - List nullLiterals = Collections.nCopies(padding, builder.literal(null)); - builder.push(input); - if (leftPadding) { - builder.project(concat(nullLiterals, builder.fields())); - } else { - builder.project(concat(builder.fields(), nullLiterals)); - } - return builder.convert(resultType, true).build(); - } - public static final RelOptRule CORRELATE_RIGHT_INSTANCE = new CorrelateRightEmptyRuleConfig() .withOperandSupplier(b0 -> b0.operand(Correlate.class).inputs( @@ -200,57 +95,6 @@ private static RelNode padWithNulls(RelBuilder builder, RelNode input, RelDataTy .withDescription("PruneEmptyCorrelate(left)") .toRule(); - /** Configuration for rule that prunes a correlate if left input is empty. */ - public static class CorrelateLeftEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5669_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - final Correlate corr = call.rel(0); - call.transformTo(call.builder().push(corr).empty().build()); - } - }; - } - } - - /** Configuration for rule that prunes a correlate if right input is empty. */ - public static class CorrelateRightEmptyRuleConfig extends HiveRuleConfig implements PruneEmptyRule.Config { - @Override - public PruneEmptyRule toRule() { - return new PruneEmptyRule(this) { - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5669_FIXED) { - throw new IllegalStateException("Class is redundant after fix is merged into Calcite"); - } - final Correlate corr = call.rel(0); - final RelNode left = call.rel(1); - final RelBuilder b = call.builder(); - final RelNode newRel; - switch (corr.getJoinType()) { - case LEFT: - newRel = padWithNulls(b, left, corr.getRowType(), false); - break; - case INNER: - case SEMI: - newRel = b.push(corr).empty().build(); - break; - case ANTI: - newRel = left; - break; - default: - throw new IllegalStateException("Correlate does not support " + corr.getJoinType()); - } - call.transformTo(newRel); - } - }; - } - } - public static final RelOptRule SORT_INSTANCE = new RemoveEmptySingleRuleConfig() .withOperandFor(HiveSortLimit.class, singleRel -> true) @@ -270,7 +114,7 @@ public void onMatch(RelOptRuleCall call) { .toRule(); public static final RelOptRule UNION_INSTANCE = - new HiveUnionEmptyPruneRuleConfig() + new UnionEmptyPruneRuleConfig() .withOperandSupplier(b0 -> b0.operand(HiveUnion.class).unorderedInputs(b1 -> b1.operand(Values.class) @@ -278,77 +122,32 @@ public void onMatch(RelOptRuleCall call) { .withDescription("HivePruneEmptyUnionBranch") .toRule(); - /** - * Copy of {@link PruneEmptyRules.UnionEmptyPruneRuleConfig} but this version expects {@link Union}. - */ - private static final class HiveUnionEmptyPruneRuleConfig extends HiveRuleConfig implements PruneEmptyRules.PruneEmptyRule.Config { - @Override - public PruneEmptyRules.PruneEmptyRule toRule() { - return new PruneEmptyRules.PruneEmptyRule(this) { - @Override public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5293_FIXED) { - throw new IllegalStateException( - "Class HiveUnionEmptyPruneRuleConfig is redundant after fix is merged into Calcite"); - } + private static final class RemoveEmptySingleRuleConfig extends HiveRuleConfig + implements RemoveEmptySingleRule.RemoveEmptySingleRuleConfig { + } - final Union union = call.rel(0); - final List inputs = union.getInputs(); - assert inputs != null; - final RelBuilder builder = call.builder(); - int nonEmptyInputs = 0; - for (RelNode input : inputs) { - if (!isEmpty(input)) { - builder.push(input); - nonEmptyInputs++; - } - } - assert nonEmptyInputs < inputs.size() - : "planner promised us at least one Empty child: " - + RelOptUtil.toString(union); - if (nonEmptyInputs == 0) { - builder.push(union).empty(); - } else { - builder.union(union.all, nonEmptyInputs); - builder.convert(union.getRowType(), true); - } - call.transformTo(builder.build()); - } - }; - } + private static final class SortFetchZeroRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.SortFetchZeroRuleConfig { } - private static boolean isEmpty(RelNode node) { - if (Bug.CALCITE_5293_FIXED) { - throw new IllegalStateException( - "Method HiveRemoveEmptySingleRules.isEmpty is redundant after fix is merged into Calcite"); - } + private static final class UnionEmptyPruneRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.UnionEmptyPruneRuleConfig { + } - if (node instanceof Values) { - return ((Values) node).getTuples().isEmpty(); - } - if (node instanceof HepRelVertex) { - return isEmpty(((HepRelVertex) node).getCurrentRel()); - } - // Note: relation input might be a RelSubset, so we just iterate over the relations - // in order to check if the subset is equivalent to an empty relation. - if (!(node instanceof RelSubset)) { - return false; - } - RelSubset subset = (RelSubset) node; - for (RelNode rel : subset.getRels()) { - if (isEmpty(rel)) { - return true; - } - } - return false; + private static final class JoinLeftEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.JoinLeftEmptyRuleConfig { } - private static final class RemoveEmptySingleRuleConfig extends HiveRuleConfig - implements RemoveEmptySingleRule.RemoveEmptySingleRuleConfig { + private static final class JoinRightEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.JoinRightEmptyRuleConfig { } - private static final class SortFetchZeroRuleConfig extends HiveRuleConfig - implements PruneEmptyRules.SortFetchZeroRuleConfig { + private static final class CorrelateLeftEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.CorrelateLeftEmptyRuleConfig { + } + + private static final class CorrelateRightEmptyRuleConfig extends HiveRuleConfig + implements PruneEmptyRules.CorrelateRightEmptyRuleConfig { } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java index 73ee02a9b355..16018ffc2f40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java @@ -18,143 +18,24 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Union; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Pair; -import org.apache.calcite.util.mapping.Mappings; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; +import org.apache.calcite.rel.rules.UnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.ImmutableList; /** * Planner rule that pulls up constants through a Union operator. */ -public class HiveUnionPullUpConstantsRule extends RelOptRule { - - protected static final Logger LOG = LoggerFactory.getLogger(HiveUnionPullUpConstantsRule.class); - +public final class HiveUnionPullUpConstantsRule { - public static final HiveUnionPullUpConstantsRule INSTANCE = - new HiveUnionPullUpConstantsRule(HiveUnion.class, - HiveRelFactories.HIVE_BUILDER); - - private HiveUnionPullUpConstantsRule( - Class unionClass, - RelBuilderFactory relBuilderFactory) { - super(operand(unionClass, any()), - relBuilderFactory, null); + private HiveUnionPullUpConstantsRule() { + throw new IllegalStateException("Instantiation not allowed"); } - @Override - public void onMatch(RelOptRuleCall call) { - if (Bug.CALCITE_5337_FIXED) { - throw new IllegalStateException("Class redundant when the fix for CALCITE-5337 is merged into Calcite"); - } - - final Union union = call.rel(0); - - final int count = union.getRowType().getFieldCount(); - if (count == 1) { - // No room for optimization since we cannot create an empty - // Project operator. - return; - } - - final RexBuilder rexBuilder = union.getCluster().getRexBuilder(); - final RelMetadataQuery mq = call.getMetadataQuery(); - final RelOptPredicateList predicates = mq.getPulledUpPredicates(union); - if (predicates == null) { - return; - } - - Map constants = new HashMap<>(); - for (int i = 0; i < count ; i++) { - RexNode expr = rexBuilder.makeInputRef(union, i); - if (predicates.constantMap.containsKey(expr)) { - constants.put(expr, predicates.constantMap.get(expr)); - } - } - - // None of the expressions are constant. Nothing to do. - if (constants.isEmpty()) { - return; - } - - // Create expressions for Project operators before and after the Union - List fields = union.getRowType().getFieldList(); - List topChildExprs = new ArrayList<>(); - List topChildExprsFields = new ArrayList<>(); - List refs = new ArrayList<>(); - ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder(); - for (int i = 0; i < count ; i++) { - RexNode expr = rexBuilder.makeInputRef(union, i); - RelDataTypeField field = fields.get(i); - if (constants.containsKey(expr)) { - if (constants.get(expr).getType().equals(field.getType())) { - topChildExprs.add(constants.get(expr)); - } else { - topChildExprs.add(rexBuilder.makeCast(field.getType(), constants.get(expr), true)); - } - topChildExprsFields.add(field.getName()); - } else { - topChildExprs.add(expr); - topChildExprsFields.add(field.getName()); - refs.add(expr); - refsIndexBuilder.set(i); - } - } - ImmutableBitSet refsIndex = refsIndexBuilder.build(); - - // Update top Project positions - final Mappings.TargetMapping mapping = - RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse(); - topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs)); - - // Create new Project-Union-Project sequences - final RelBuilder relBuilder = call.builder(); - for (int i = 0; i < union.getInputs().size() ; i++) { - RelNode input = union.getInput(i); - List> newChildExprs = new ArrayList<>(); - for (int j = 0; j < refsIndex.cardinality(); j++) { - int pos = refsIndex.nth(j); - newChildExprs.add(Pair.of(rexBuilder.makeInputRef(input, pos), - input.getRowType().getFieldList().get(pos).getName())); - } - if (newChildExprs.isEmpty()) { - // At least a single item in project is required. - newChildExprs.add(Pair.of(topChildExprs.get(0), topChildExprsFields.get(0))); - } - // Add the input with project on top - relBuilder.push(input); - relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs)); - } - relBuilder.union(union.all, union.getInputs().size()); - // Create top Project fixing nullability of fields - relBuilder.project(topChildExprs, topChildExprsFields); - relBuilder.convert(union.getRowType(), false); - - call.transformTo(relBuilder.build()); - } + public static final RelOptRule INSTANCE = + UnionPullUpConstantsRule.Config.DEFAULT + .withOperandFor(HiveUnion.class) + .withRelBuilderFactory(HiveRelFactories.HIVE_BUILDER) + .toRule(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 6e0430f4019e..9a9f48030a59 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -22,124 +22,28 @@ import java.util.List; import java.util.Set; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.rel.SingleRel; -import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.MetadataDef; import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Util; -import org.apache.hadoop.hive.ql.optimizer.calcite.Bug; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; - public class HiveRelMdUniqueKeys implements MetadataHandler { public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( BuiltInMethod.UNIQUE_KEYS.method, new HiveRelMdUniqueKeys()); - /** - * A limit about the number of unique keys returned by the handler. - * The limit must be in the range [0, Integer.MAX_VALUE]. - */ - private final int limit; - - private HiveRelMdUniqueKeys() { - if (Bug.CALCITE_6704_FIXED) { - throw new IllegalStateException("Remove constructor and limit once we upgrade to version with CALCITE-6704"); - } - this.limit = 1000; - } @Override public MetadataDef getDef() { return BuiltInMetadata.UniqueKeys.DEF; } - public Set getUniqueKeys(Project rel, RelMetadataQuery mq, - boolean ignoreNulls) { - return getProjectUniqueKeys(rel, mq, ignoreNulls, rel.getProjects()); - } - - private Set getProjectUniqueKeys(SingleRel rel, RelMetadataQuery mq, - boolean ignoreNulls, List projExprs) { - if (Bug.CALCITE_6704_FIXED) { - throw new IllegalStateException("Method is redundant once we upgrade to version with CALCITE-6704"); - - } - // LogicalProject maps a set of rows to a different set; - // Without knowledge of the mapping function(whether it - // preserves uniqueness), it is only safe to derive uniqueness - // info from the child of a project when the mapping is f(a) => a. - // - // Further more, the unique bitset coming from the child needs - // to be mapped to match the output of the project. - - // Single input can be mapped to multiple outputs - ImmutableMultimap.Builder inToOutPosBuilder = ImmutableMultimap.builder(); - ImmutableBitSet.Builder mappedInColumnsBuilder = ImmutableBitSet.builder(); - - // Build an input to output position map. - for (int i = 0; i < projExprs.size(); i++) { - RexNode projExpr = projExprs.get(i); - if (projExpr instanceof RexInputRef) { - int inputIndex = ((RexInputRef) projExpr).getIndex(); - inToOutPosBuilder.put(inputIndex, i); - mappedInColumnsBuilder.set(inputIndex); - } - } - ImmutableBitSet inColumnsUsed = mappedInColumnsBuilder.build(); - - if (inColumnsUsed.isEmpty()) { - // if there's no RexInputRef in the projected expressions - // return empty set. - return ImmutableSet.of(); - } - - Set childUniqueKeySet = - mq.getUniqueKeys(rel.getInput(), ignoreNulls); - - if (childUniqueKeySet == null) { - return ImmutableSet.of(); - } - - Multimap mapInToOutPos = inToOutPosBuilder.build(); - - Set resultBuilder = new HashSet<>(); - // Now add to the projUniqueKeySet the child keys that are fully - // projected. - outerLoop: - for (ImmutableBitSet colMask : childUniqueKeySet) { - if (!inColumnsUsed.contains(colMask)) { - // colMask contains a column that is not projected as RexInput => the key is not unique - continue; - } - // colMask is mapped to output project, however, the column can be mapped more than once: - // select key1, key1, val1, val2, key2 from ... - // the resulting unique keys would be {{0},{4}}, {{1},{4}} - - Iterable> product = Linq4j.product(Util.transform(colMask.toList(), mapInToOutPos::get)); - for (List passKey : product) { - if (resultBuilder.size() == limit) { - break outerLoop; - } - resultBuilder.add(ImmutableBitSet.of(passKey)); - } - } - return resultBuilder; - } - public Set getUniqueKeys(HiveTableScan rel, RelMetadataQuery mq, boolean ignoreNulls) { RelOptHiveTable tbl = (RelOptHiveTable) rel.getTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 819ff93dbb58..f370f20c626f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -60,6 +60,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.RuleEventLogger; import org.apache.calcite.plan.hep.HepMatchOrder; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; @@ -168,7 +169,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveMaterializedViewASTSubQueryRewriteShuttle; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveSqlTypeUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.RuleEventLogger; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.CteRuleConfig; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateSortLimitRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinSwapConstraintsRule;