diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index b83e0a88a5e2..7350a92a07f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -665,7 +665,7 @@ JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work) HiveOperation operation = queryState.getHiveOperation(); JSONObject object = new JSONObject(new LinkedHashMap<>()); - Object jsonInput = toJson("INPUTS", toString(analyzer.getInputs()), out, work); + Object jsonInput = toJson("INPUTS", toString(analyzer.getAllInputs()), out, work); if (work.isFormatted()) { object.put("INPUTS", jsonInput); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 7d83425aff82..93d94a929f0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1649,7 +1649,7 @@ public boolean hasTransactionalInQuery() { public boolean isRequiresOpenTransaction() { return hasTransactionalInQuery() || getAcidDdlDesc() != null || - Stream.of(getInputs(), getOutputs()).flatMap(Collection::stream) + Stream.of(getAllInputs(), getOutputs()).flatMap(Collection::stream) .filter(entity -> entity.getType() == Entity.Type.TABLE || entity.getType() == Entity.Type.PARTITION) .flatMap(entity -> { Table tbl = entity.getTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java index 1d80a8643709..5a7f16b2f670 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java @@ -34,7 +34,8 @@ public ColumnAccessAnalyzer(ParseContext pactx) { pGraphContext = pactx; } - public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo columnAccessInfo) throws SemanticException { + public ColumnAccessInfo analyzeColumnAccess(SemanticAnalyzer analyzer) throws SemanticException { + ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); if (columnAccessInfo == null) { columnAccessInfo = new ColumnAccessInfo(); } @@ -58,6 +59,21 @@ public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo columnAccessInfo) t } } } + columnAccessInfo.merge(getMaterializedCteColumnAccessInfo(analyzer.rootClause.asExecutionOrder())); + return columnAccessInfo; + } + + /** + * Merge column access recorded by materialized CTE sub-analyzers into this analyzer's + * column access info so authorization sees base-table columns, not only the temp CTE table. + */ + private ColumnAccessInfo getMaterializedCteColumnAccessInfo(List cteClauses) { + ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo(); + for (SemanticAnalyzer.CTEClause cte : cteClauses) { + if (cte.source != null && cte.source.getColumnAccessInfo() != null) { + columnAccessInfo.merge(cte.source.getColumnAccessInfo()); + } + } return columnAccessInfo; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java index ca1c39174660..ea790218eeeb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java @@ -92,6 +92,16 @@ public Map> getTableToColumnAllAccessMap() { return mapping; } + /** + * Merge column accesses from another ColumnAccessInfo into this one. + */ + public void merge(ColumnAccessInfo other) { + if (other == null) { + return; + } + tableToColumnAccessMap.putAll(other.tableToColumnAccessMap); + } + /** * Strip a virtual column out of the set of columns. This is useful in cases where we do not * want to be checking against the user reading virtual columns, namely update and delete. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java index ab852aa03ff1..ba7694ce7daa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java @@ -133,7 +133,7 @@ private boolean tryMetadataUpdate(Table table, ASTNode tabNameNode, ASTNode wher DDLWork ddlWork = createDDLWorkOfMetadataUpdate(tableName, sarg); rootTasks = Collections.singletonList(TaskFactory.get(ddlWork)); - inputs = sem.getInputs(); + inputs = sem.getAllInputs(); outputs = sem.getOutputs(); updateOutputs(table); return true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index d6175ccb2c27..243646130210 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -186,7 +186,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, input); sem.analyze(input, ctx); sem.validate(); - inputs = sem.getInputs(); + inputs = sem.getAllInputs(); outputs = sem.getOutputs(); ctx.setResFile(ctx.getLocalTmpPath()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index e28dc4751195..1c951f9df020 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -56,7 +56,7 @@ public void setConf(Configuration conf) { @Override public void update(BaseSemanticAnalyzer sem) { - this.inputs = sem.getInputs(); + this.inputs = sem.getAllInputs(); this.outputs = sem.getOutputs(); this.commandType = sem.getQueryState().getHiveOperation(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7042cff53874..8e4c781a882a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1406,7 +1406,7 @@ private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr); } - private final CTEClause rootClause = new CTEClause(null, null, null); + final CTEClause rootClause = new CTEClause(null, null, null); @Override public List> getAllRootTasks() { @@ -1421,10 +1421,10 @@ public List> getAllRootTasks() { @Override public Set getAllInputs() { - Set readEntities = new HashSet(getInputs()); + Set readEntities = new LinkedHashSet<>(getInputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { - readEntities.addAll(cte.source.getInputs()); + readEntities.addAll(cte.source.getAllInputs()); } } return readEntities; @@ -1435,7 +1435,7 @@ public Set getAllOutputs() { Set writeEntities = new HashSet(getOutputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { - writeEntities.addAll(cte.source.getOutputs()); + writeEntities.addAll(cte.source.getAllOutputs()); } } return writeEntities; @@ -1595,9 +1595,8 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { LOG.info("{} will be materialized into {}", cteName, location); cte.source = analyzer; - + ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp())); - return table; } @@ -13387,7 +13386,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE || HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) { ColumnAccessAnalyzer columnAccessAnalyzer = new ColumnAccessAnalyzer(pCtx); // view column access info is carried by this.getColumnAccessInfo(). - setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this.getColumnAccessInfo())); + setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this)); } } perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.LOGICAL_OPTIMIZATION); @@ -13426,7 +13425,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE // 11. put accessed columns to readEntity if (HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) { - putAccessedColumnsToReadEntity(inputs, columnAccessInfo); + putAccessedColumnsToReadEntity(getAllInputs(), columnAccessInfo); } if (isCacheEnabled && lookupInfo != null) { @@ -15288,7 +15287,7 @@ private void useCachedResult(QueryResultsCache.CacheEntry cacheEntry, boolean ne private QueryResultsCache.QueryInfo createCacheQueryInfoForQuery(QueryResultsCache.LookupInfo lookupInfo) { long queryTime = SessionState.get().getQueryCurrentTimestamp().toEpochMilli(); return new QueryResultsCache.QueryInfo(queryTime, lookupInfo, queryState.getHiveOperation(), - resultSchema, getTableAccessInfo(), getColumnAccessInfo(), inputs); + resultSchema, getTableAccessInfo(), getColumnAccessInfo(), getAllInputs()); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java index dda18f0649d8..6d9abeb979f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java @@ -81,7 +81,7 @@ public ExplainWork(Path resFile, } this.analyzer = analyzer; if (analyzer != null) { - this.inputs = analyzer.getInputs(); + this.inputs = analyzer.getAllInputs(); } if (analyzer != null) { this.outputs = analyzer.getAllOutputs(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java index cc7b49a05881..98f95fb37df1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java @@ -75,14 +75,15 @@ private static boolean skip(HiveOperation op, BaseSemanticAnalyzer sem) throws H private static Set getInputs(BaseSemanticAnalyzer sem) { Set additionalInputs = new HashSet(); - for (Entity e : sem.getInputs()) { + for (Entity e : sem.getAllInputs()) { if (e.getType() == Entity.Type.PARTITION) { additionalInputs.add(new ReadEntity(e.getTable())); } } + // getAllInputs() includes tables read by materialized CTE sub-analyzers. // Sets.union keeps the values from the first set if they are present in both - return Sets.union(sem.getInputs(), additionalInputs); + return Sets.union(sem.getAllInputs(), additionalInputs); } private static Set getOutputs(BaseSemanticAnalyzer sem) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java new file mode 100644 index 000000000000..6a888cb3131b --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.database.drop.DropDatabaseDesc; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestMaterializedCTEInputs { + private static final String DB_NAME = "test_materialized_cte"; + private static final String TABLE_FQ_NAME = DB_NAME + "@src"; + + private static Hive db; + private static HiveConf conf; + + @BeforeAll + public static void beforeClass() throws Exception { + conf = new HiveConfForTest(TestMaterializedCTEInputs.class); + conf.set("hive.security.authorization.enabled", "false"); + conf.set("hive.security.authorization.manager", + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory"); + conf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 1); + conf.setBoolVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_FULL_AGGREGATE_ONLY, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); + db = Hive.get(conf); + SessionState.start(conf); + new DatabaseBuilder().setName(DB_NAME).create(db.getMSC(), conf); + SessionState.get().setCurrentDatabase(DB_NAME); + new TableBuilder().setDbName(DB_NAME).setTableName("src") + .addCol("key", "string") + .addCol("value", "string") + .addCol("col1", "int") + .create(db.getMSC(), conf); + } + + public static Stream casesForMaterializedCteInputs() { + return Stream.of( + Arguments.of("chain cte", "with q1 as ( select key from q2 where key = '5')," + + "q2 as ( select key from test_materialized_cte.src where key = '5') " + + "select * from (select key from q1) a", Set.of("key")), + Arguments.of("nested cte", "WITH q1 AS (" + + "WITH q2 AS (SELECT key, value FROM test_materialized_cte.src WHERE key = '4') " + + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " + + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key", Set.of("key", "value")), + Arguments.of("merge columns", "WITH q1 AS (" + + "WITH q2 AS (SELECT key FROM test_materialized_cte.src WHERE key = '4') " + + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " + + "SELECT * FROM q1 t1 JOIN test_materialized_cte.src t2 ON t1.key = t2.key", Set.of("key", "value")) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("casesForMaterializedCteInputs") + void testMaterializedCteInputs(String type, String query, Set expectedCols) throws Exception { + HiveConf testConf = new HiveConf(conf); + Context ctx = new Context(testConf); + ASTNode astNode = ParseUtils.parse(query, ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); + SemanticAnalyzer analyzer = (SemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + analyzer.analyze(astNode, ctx); + + Set allInputs = analyzer.getAllInputs(); + + assertTrue(allInputs.stream().anyMatch(e -> isTableNamed(e, "src")), + "Materialized CTE base table must appear in getAllInputs"); + + ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); + assertNotNull(columnAccessInfo); + List srcCols = columnAccessInfo.getTableToColumnAccessMap().get(TABLE_FQ_NAME); + assertNotNull(srcCols, "Column must include materialized CTE base table"); + assertTrue(new HashSet<>(srcCols).containsAll(expectedCols), + () -> "Expected columns " + expectedCols + " but got " + srcCols); + ctx.clear(); + } + + private static boolean isTableNamed(ReadEntity entity, String tableName) { + return entity.getTable() != null && tableName.equals(entity.getTable().getTableName()); + } + + @AfterAll + public static void afterClass() throws Exception { + try { + db.dropDatabase(new DropDatabaseDesc(DB_NAME, DB_NAME, true, true, true)); + } catch (Exception ignored) { + } + db.close(true); + } +} diff --git a/ql/src/test/results/clientpositive/llap/cte_3.q.out b/ql/src/test/results/clientpositive/llap/cte_3.q.out index ee1fe65effff..1a7f931ce039 100644 --- a/ql/src/test/results/clientpositive/llap/cte_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -4,6 +4,7 @@ select * from q1 PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from src where key = '5') @@ -11,6 +12,7 @@ select * from q1 POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -63,12 +65,14 @@ with q1 as ( select key from src where key = '5') select * from (select key from q1) a PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from src where key = '5') select * from (select key from q1) a POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -120,6 +124,8 @@ q2 as ( select key from src where key = '5') select * from (select key from q1) a PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from q2 where key = '5'), @@ -127,6 +133,8 @@ q2 as ( select key from src where key = '5') select * from (select key from q1) a POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out index f4cadc024bfe..809fadf40429 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out @@ -48,7 +48,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src @@ -62,7 +64,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -272,7 +276,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src @@ -286,7 +292,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -310,7 +318,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src @@ -324,7 +334,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -493,7 +505,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src @@ -507,7 +521,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -577,7 +593,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -591,7 +609,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -801,7 +821,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -815,7 +837,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -839,7 +863,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -853,7 +879,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1022,7 +1050,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -1036,7 +1066,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -1118,7 +1150,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1132,7 +1166,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1342,7 +1378,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1356,7 +1394,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -1380,7 +1420,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1394,7 +1436,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1563,7 +1607,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1577,7 +1623,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -1659,7 +1707,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1673,7 +1723,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1883,7 +1935,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1897,7 +1951,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -1921,7 +1977,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1935,7 +1993,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -2104,7 +2164,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -2118,7 +2180,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out index d63d003dc266..955bd1779023 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -5,6 +5,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -13,6 +14,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out index 4a1d6e0b06a9..0c6a5384f66f 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -30,6 +30,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -38,6 +39,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -154,6 +156,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -162,6 +165,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out index c5f5810ec36e..506e9b7a83bb 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -48,6 +48,7 @@ from mydb.q1 as a join q1 as b on a.colnum=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src PREHOOK: Input: mydb@q1 #### A masked pattern was here #### POSTHOOK: query: explain @@ -57,6 +58,7 @@ from mydb.q1 as a join q1 as b on a.colnum=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src POSTHOOK: Input: mydb@q1 #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out index a5064535c42f..363b96322fa9 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out @@ -76,6 +76,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@b1 PREHOOK: Input: default@b2 PREHOOK: Input: default@cte_mat_9_a +PREHOOK: Input: default@cte_mat_9_b #### A masked pattern was here #### POSTHOOK: query: explain with a0 AS ( select id, 'a0' as tag from cte_mat_9_a @@ -108,6 +109,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@b1 POSTHOOK: Input: default@b2 POSTHOOK: Input: default@cte_mat_9_a +POSTHOOK: Input: default@cte_mat_9_b #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out index 2ffb09e431fc..0fe1b644e3cc 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out @@ -50,6 +50,8 @@ FROM dept_avg da WHERE da.avg_salary < 100000 PREHOOK: type: QUERY PREHOOK: Input: default@dept_avg +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH dept_avg AS ( @@ -67,6 +69,8 @@ FROM dept_avg da WHERE da.avg_salary < 100000 POSTHOOK: type: QUERY POSTHOOK: Input: default@dept_avg +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps #### A masked pattern was here #### CBO PLAN: HiveProject(d_name=[$0], _c1=[$1])