From 77aebb40c7c792f26d2e57078450fe76998ba9d6 Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 13 Aug 2026 09:56:37 +0800 Subject: [PATCH 1/7] HIVE-29807 --- .../hadoop/hive/ql/exec/ExplainTask.java | 2 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hive/ql/parse/ColumnAccessAnalyzer.java | 19 ++++++- .../hive/ql/parse/ColumnAccessInfo.java | 14 +++++ .../hive/ql/parse/DeleteSemanticAnalyzer.java | 2 +- .../ql/parse/ExplainSemanticAnalyzer.java | 2 +- .../HiveSemanticAnalyzerHookContextImpl.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 17 +++--- .../hadoop/hive/ql/plan/ExplainWork.java | 2 +- .../command/CommandAuthorizer.java | 10 ++-- .../command/CommandAuthorizerV1.java | 8 ++- .../hive/ql/parse/TestSemanticAnalyzer.java | 57 +++++++++++++++++++ 12 files changed, 115 insertions(+), 22 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index b83e0a88a5e2..7350a92a07f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -665,7 +665,7 @@ JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work) HiveOperation operation = queryState.getHiveOperation(); JSONObject object = new JSONObject(new LinkedHashMap<>()); - Object jsonInput = toJson("INPUTS", toString(analyzer.getInputs()), out, work); + Object jsonInput = toJson("INPUTS", toString(analyzer.getAllInputs()), out, work); if (work.isFormatted()) { object.put("INPUTS", jsonInput); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 7d83425aff82..93d94a929f0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1649,7 +1649,7 @@ public boolean hasTransactionalInQuery() { public boolean isRequiresOpenTransaction() { return hasTransactionalInQuery() || getAcidDdlDesc() != null || - Stream.of(getInputs(), getOutputs()).flatMap(Collection::stream) + Stream.of(getAllInputs(), getOutputs()).flatMap(Collection::stream) .filter(entity -> entity.getType() == Entity.Type.TABLE || entity.getType() == Entity.Type.PARTITION) .flatMap(entity -> { Table tbl = entity.getTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java index 1d80a8643709..3200dca136a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java @@ -34,7 +34,8 @@ public ColumnAccessAnalyzer(ParseContext pactx) { pGraphContext = pactx; } - public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo columnAccessInfo) throws SemanticException { + public ColumnAccessInfo analyzeColumnAccess(SemanticAnalyzer analyzer) throws SemanticException { + ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); if (columnAccessInfo == null) { columnAccessInfo = new ColumnAccessInfo(); } @@ -58,6 +59,22 @@ public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo columnAccessInfo) t } } } + // Every Analyzer holds its private rootClause + columnAccessInfo.merge(getMaterializedCteColumnAccessInfo(analyzer.rootClause.asExecutionOrder())); + return columnAccessInfo; + } + + /** + * Merge column access recorded by materialized CTE sub-analyzers into this analyzer's + * column access info so authorization sees base-table columns, not only the temp CTE table. + */ + private ColumnAccessInfo getMaterializedCteColumnAccessInfo(List cteClauses) { + ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo(); + for (SemanticAnalyzer.CTEClause cte : cteClauses) { + if (cte.source != null && cte.source.getColumnAccessInfo() != null) { + columnAccessInfo.merge(cte.source.getColumnAccessInfo()); + } + } return columnAccessInfo; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java index ca1c39174660..d06a06558d31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java @@ -92,6 +92,20 @@ public Map> getTableToColumnAllAccessMap() { return mapping; } + /** + * Merge direct column accesses from another ColumnAccessInfo into this one. + */ + public void merge(ColumnAccessInfo other) { + if (other == null) { + return; + } + for (Map.Entry> entry : other.getTableToColumnAccessMap().entrySet()) { + for (String col : entry.getValue()) { + add(entry.getKey(), col); + } + } + } + /** * Strip a virtual column out of the set of columns. This is useful in cases where we do not * want to be checking against the user reading virtual columns, namely update and delete. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java index ab852aa03ff1..ba7694ce7daa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java @@ -133,7 +133,7 @@ private boolean tryMetadataUpdate(Table table, ASTNode tabNameNode, ASTNode wher DDLWork ddlWork = createDDLWorkOfMetadataUpdate(tableName, sarg); rootTasks = Collections.singletonList(TaskFactory.get(ddlWork)); - inputs = sem.getInputs(); + inputs = sem.getAllInputs(); outputs = sem.getOutputs(); updateOutputs(table); return true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index d6175ccb2c27..243646130210 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -186,7 +186,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, input); sem.analyze(input, ctx); sem.validate(); - inputs = sem.getInputs(); + inputs = sem.getAllInputs(); outputs = sem.getOutputs(); ctx.setResFile(ctx.getLocalTmpPath()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index e28dc4751195..1c951f9df020 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -56,7 +56,7 @@ public void setConf(Configuration conf) { @Override public void update(BaseSemanticAnalyzer sem) { - this.inputs = sem.getInputs(); + this.inputs = sem.getAllInputs(); this.outputs = sem.getOutputs(); this.commandType = sem.getQueryState().getHiveOperation(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7042cff53874..074e11b85d58 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1406,7 +1406,7 @@ private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr); } - private final CTEClause rootClause = new CTEClause(null, null, null); + final CTEClause rootClause = new CTEClause(null, null, null); @Override public List> getAllRootTasks() { @@ -1424,7 +1424,7 @@ public Set getAllInputs() { Set readEntities = new HashSet(getInputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { - readEntities.addAll(cte.source.getInputs()); + readEntities.addAll(cte.source.getAllInputs()); } } return readEntities; @@ -1435,7 +1435,7 @@ public Set getAllOutputs() { Set writeEntities = new HashSet(getOutputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { - writeEntities.addAll(cte.source.getOutputs()); + writeEntities.addAll(cte.source.getAllOutputs()); } } return writeEntities; @@ -1595,9 +1595,8 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { LOG.info("{} will be materialized into {}", cteName, location); cte.source = analyzer; - + ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp())); - return table; } @@ -13312,7 +13311,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject, opToSamplePruner, - globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, + globalLimitCtx, nameToSplitSample, getAllInputs(), rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, analyzeRewrite, tableDesc, createVwDesc, materializedViewUpdateDesc, queryProperties, viewProjectToTableSchema); @@ -13387,7 +13386,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE || HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) { ColumnAccessAnalyzer columnAccessAnalyzer = new ColumnAccessAnalyzer(pCtx); // view column access info is carried by this.getColumnAccessInfo(). - setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this.getColumnAccessInfo())); + setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this)); } } perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.LOGICAL_OPTIMIZATION); @@ -13426,7 +13425,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE // 11. put accessed columns to readEntity if (HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) { - putAccessedColumnsToReadEntity(inputs, columnAccessInfo); + putAccessedColumnsToReadEntity(getAllInputs(), columnAccessInfo); } if (isCacheEnabled && lookupInfo != null) { @@ -15288,7 +15287,7 @@ private void useCachedResult(QueryResultsCache.CacheEntry cacheEntry, boolean ne private QueryResultsCache.QueryInfo createCacheQueryInfoForQuery(QueryResultsCache.LookupInfo lookupInfo) { long queryTime = SessionState.get().getQueryCurrentTimestamp().toEpochMilli(); return new QueryResultsCache.QueryInfo(queryTime, lookupInfo, queryState.getHiveOperation(), - resultSchema, getTableAccessInfo(), getColumnAccessInfo(), inputs); + resultSchema, getTableAccessInfo(), getColumnAccessInfo(), getAllInputs()); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java index dda18f0649d8..6d9abeb979f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java @@ -81,7 +81,7 @@ public ExplainWork(Path resFile, } this.analyzer = analyzer; if (analyzer != null) { - this.inputs = analyzer.getInputs(); + this.inputs = analyzer.getAllInputs(); } if (analyzer != null) { this.outputs = analyzer.getAllOutputs(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java index cc7b49a05881..3b11a41e35f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java @@ -75,25 +75,27 @@ private static boolean skip(HiveOperation op, BaseSemanticAnalyzer sem) throws H private static Set getInputs(BaseSemanticAnalyzer sem) { Set additionalInputs = new HashSet(); - for (Entity e : sem.getInputs()) { + for (Entity e : sem.getAllInputs()) { if (e.getType() == Entity.Type.PARTITION) { additionalInputs.add(new ReadEntity(e.getTable())); } } + // getAllInputs() includes tables read by materialized CTE sub-analyzers. // Sets.union keeps the values from the first set if they are present in both - return Sets.union(sem.getInputs(), additionalInputs); + return Sets.union(sem.getAllInputs(), additionalInputs); } private static Set getOutputs(BaseSemanticAnalyzer sem) { Set additionalOutputs = new HashSet(); - for (WriteEntity e : sem.getOutputs()) { + for (WriteEntity e : sem.getAllOutputs()) { if (e.getType() == Entity.Type.PARTITION) { additionalOutputs.add(new WriteEntity(e.getTable(), e.getWriteType())); } } + // getAllOutputs() includes writes performed by materialized CTE sub-analyzers. // Sets.union keeps the values from the first set if they are present in both - return Sets.union(sem.getOutputs(), additionalOutputs); + return Sets.union(sem.getAllOutputs(), additionalOutputs); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java index 73630048532a..d6629112e1f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java @@ -159,7 +159,7 @@ private static void authorizeInputs(HiveOperation op, BaseSemanticAnalyzer sem, } } - authorizeTable(op, authorizer, tableUsePartLevelAuth, tab2Cols, tableAuthChecked, tbl); + authorizeTable(op, sem, authorizer, tableUsePartLevelAuth, tab2Cols, tableAuthChecked, tbl); } } @@ -232,7 +232,8 @@ private static void getTablePartitionUsedColumns(HiveOperation op, BaseSemanticA } } - private static void authorizeTable(HiveOperation op, HiveAuthorizationProvider authorizer, + private static void authorizeTable(HiveOperation op, BaseSemanticAnalyzer sem, + HiveAuthorizationProvider authorizer, Map tableUsePartLevelAuth, Map> tab2Cols, Set tableAuthChecked, Table tbl) throws HiveException { // if we reach here, it means it needs to do a table authorization check, and the table authorization may @@ -240,6 +241,9 @@ private static void authorizeTable(HiveOperation op, HiveAuthorizationProvider a if (tbl != null && !tableAuthChecked.contains(tbl.getTableName()) && !(Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName())))) { List cols = tab2Cols.get(tbl); + if ((cols == null || cols.isEmpty()) && sem.getColumnAccessInfo() != null) { + cols = sem.getColumnAccessInfo().getTableToColumnAccessMap().get(tbl.getCompleteName()); + } if (cols != null && cols.size() > 0) { authorizer.authorize(tbl, null, cols, op.getInputRequiredPrivileges(), null); } else { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index 17ac1b353bf2..d13eb7617436 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -545,4 +546,60 @@ private void testMaterializeCTEUsesDDLFactory(boolean cboEnabled) throws Excepti cteAnalyzer[0] instanceof CreateTableAnalyzer); } } + + @Test + public void testMaterializedCteInputsAndColumnAccess() throws Exception { + createKeyValueTable("src"); + + HiveConf testConf = new HiveConf(conf); + testConf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 1); + testConf.setBoolVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_FULL_AGGREGATE_ONLY, false); + testConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); + + SessionState.start(testConf); + String[] queries = { + "with q1 as ( select key from q2 where key = '5')," + + "q2 as ( select key from src where key = '5') " + + "select * from (select key from q1) a", + "WITH q1 AS (" + + "WITH q2 AS (SELECT key, value FROM src WHERE key = '4') " + + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " + + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key" + }; + + SemanticAnalyzer[] analyzers = new SemanticAnalyzer[queries.length]; + for (int i = 0; i < queries.length; i++) { + Context ctx = new Context(testConf); + ASTNode astNode = ParseUtils.parse(queries[i], ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); + SemanticAnalyzer analyzer = (SemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + analyzer.analyze(astNode, ctx); + analyzers[i] = analyzer; + } + + for (int i = 0; i < analyzers.length; i++) { + SemanticAnalyzer analyzer = analyzers[i]; + Set directInputs = analyzer.getInputs(); + Set allInputs = analyzer.getAllInputs(); + + assertTrue("Materialized CTE should not expose base table in direct inputs", + directInputs.stream().noneMatch(e -> isTableNamed(e, "src"))); + assertTrue("Nested materialized CTE base table must appear in getAllInputs", + allInputs.stream().anyMatch(e -> isTableNamed(e, "src"))); + + ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); + assertNotNull(columnAccessInfo); + List srcCols = columnAccessInfo.getTableToColumnAccessMap().get("default@src"); + assertNotNull("Column must include nested materialized CTE base table", srcCols); + assertTrue(srcCols.contains("key")); + if ( i == analyzers.length - 1) { + assertTrue(srcCols.contains("value")); + } + } + } + + private static boolean isTableNamed(ReadEntity entity, String tableName) { + return entity.getTable() != null && tableName.equals(entity.getTable().getTableName()); + } } From 71ade86f6c95d54abd019fddabfab419b0009891 Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 13 Aug 2026 17:58:45 +0800 Subject: [PATCH 2/7] fix --- .../java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 074e11b85d58..ac9d6aa7bd45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -13311,7 +13311,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject, opToSamplePruner, - globalLimitCtx, nameToSplitSample, getAllInputs(), rootTasks, opToPartToSkewedPruner, + globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, analyzeRewrite, tableDesc, createVwDesc, materializedViewUpdateDesc, queryProperties, viewProjectToTableSchema); From 0b6edbe45ef90fcf795077cc4659d7838664bb43 Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 13 Aug 2026 18:12:08 +0800 Subject: [PATCH 3/7] qfile output --- .../results/clientpositive/llap/cte_3.q.out | 8 +++ .../clientpositive/llap/cte_mat_11.q.out | 64 +++++++++++++++++++ .../clientpositive/llap/cte_mat_3.q.out | 2 + .../clientpositive/llap/cte_mat_4.q.out | 4 ++ .../clientpositive/llap/cte_mat_5.q.out | 2 + .../clientpositive/llap/cte_mat_9.q.out | 2 + .../clientpositive/llap/cte_mat_type.q.out | 4 ++ 7 files changed, 86 insertions(+) diff --git a/ql/src/test/results/clientpositive/llap/cte_3.q.out b/ql/src/test/results/clientpositive/llap/cte_3.q.out index ee1fe65effff..1a7f931ce039 100644 --- a/ql/src/test/results/clientpositive/llap/cte_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -4,6 +4,7 @@ select * from q1 PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from src where key = '5') @@ -11,6 +12,7 @@ select * from q1 POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -63,12 +65,14 @@ with q1 as ( select key from src where key = '5') select * from (select key from q1) a PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from src where key = '5') select * from (select key from q1) a POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -120,6 +124,8 @@ q2 as ( select key from src where key = '5') select * from (select key from q1) a PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as ( select key from q2 where key = '5'), @@ -127,6 +133,8 @@ q2 as ( select key from src where key = '5') select * from (select key from q1) a POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out index f4cadc024bfe..809fadf40429 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out @@ -48,7 +48,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src @@ -62,7 +64,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -272,7 +276,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src @@ -286,7 +292,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -310,7 +318,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src @@ -324,7 +334,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -493,7 +505,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src @@ -507,7 +521,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -577,7 +593,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -591,7 +609,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -801,7 +821,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -815,7 +837,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -839,7 +863,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -853,7 +879,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1022,7 +1050,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_no_stats #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_no_stats @@ -1036,7 +1066,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_no_stats #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -1118,7 +1150,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1132,7 +1166,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1342,7 +1378,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1356,7 +1394,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -1380,7 +1420,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1394,7 +1436,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1563,7 +1607,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_key @@ -1577,7 +1623,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_key #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) @@ -1659,7 +1707,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1673,7 +1723,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1883,7 +1935,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1897,7 +1951,9 @@ SELECT a.key FROM materialized_cte2 a JOIN materialized_cte2 b ON (a.key = b.key) POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### CBO PLAN: HiveProject(a.key=[$0]) @@ -1921,7 +1977,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -1935,7 +1993,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage @@ -2104,7 +2164,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 PREHOOK: type: QUERY +PREHOOK: Input: default@materialized_cte1 PREHOOK: Input: default@materialized_cte2 +PREHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS ( SELECT * FROM src_partial_stats_value @@ -2118,7 +2180,9 @@ SELECT * FROM materialized_cte2 UNION ALL SELECT * FROM materialized_cte2 POSTHOOK: type: QUERY +POSTHOOK: Input: default@materialized_cte1 POSTHOOK: Input: default@materialized_cte2 +POSTHOOK: Input: default@src_partial_stats_value #### A masked pattern was here #### CBO PLAN: HiveProject(key=[$0], value=[$1]) diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out index d63d003dc266..955bd1779023 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -5,6 +5,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -13,6 +14,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out index 4a1d6e0b06a9..0c6a5384f66f 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -30,6 +30,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -38,6 +39,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. @@ -154,6 +156,7 @@ from q1 a join q1 b on a.key=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: explain with q1 as (select * from src where key= '5') @@ -162,6 +165,7 @@ from q1 a join q1 b on a.key=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out index c5f5810ec36e..506e9b7a83bb 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -48,6 +48,7 @@ from mydb.q1 as a join q1 as b on a.colnum=b.key PREHOOK: type: QUERY PREHOOK: Input: default@q1 +PREHOOK: Input: default@src PREHOOK: Input: mydb@q1 #### A masked pattern was here #### POSTHOOK: query: explain @@ -57,6 +58,7 @@ from mydb.q1 as a join q1 as b on a.colnum=b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src POSTHOOK: Input: mydb@q1 #### A masked pattern was here #### Plan optimized by CBO. diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out index a5064535c42f..363b96322fa9 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out @@ -76,6 +76,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@b1 PREHOOK: Input: default@b2 PREHOOK: Input: default@cte_mat_9_a +PREHOOK: Input: default@cte_mat_9_b #### A masked pattern was here #### POSTHOOK: query: explain with a0 AS ( select id, 'a0' as tag from cte_mat_9_a @@ -108,6 +109,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@b1 POSTHOOK: Input: default@b2 POSTHOOK: Input: default@cte_mat_9_a +POSTHOOK: Input: default@cte_mat_9_b #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out index 2ffb09e431fc..0fe1b644e3cc 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out @@ -50,6 +50,8 @@ FROM dept_avg da WHERE da.avg_salary < 100000 PREHOOK: type: QUERY PREHOOK: Input: default@dept_avg +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps #### A masked pattern was here #### POSTHOOK: query: EXPLAIN CBO WITH dept_avg AS ( @@ -67,6 +69,8 @@ FROM dept_avg da WHERE da.avg_salary < 100000 POSTHOOK: type: QUERY POSTHOOK: Input: default@dept_avg +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps #### A masked pattern was here #### CBO PLAN: HiveProject(d_name=[$0], _c1=[$1]) From 5e2b25ea4a96e007b7b16bbd291acdac4d632e6a Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 13 Aug 2026 22:23:30 +0800 Subject: [PATCH 4/7] fix --- .../java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ac9d6aa7bd45..8e4c781a882a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1421,7 +1421,7 @@ public List> getAllRootTasks() { @Override public Set getAllInputs() { - Set readEntities = new HashSet(getInputs()); + Set readEntities = new LinkedHashSet<>(getInputs()); for (CTEClause cte : rootClause.asExecutionOrder()) { if (cte.source != null) { readEntities.addAll(cte.source.getAllInputs()); From 6cf9f4a6755ee71e7f84e6b3b3c0409b507d1bd0 Mon Sep 17 00:00:00 2001 From: zdeng Date: Tue, 18 Aug 2026 20:49:12 +0800 Subject: [PATCH 5/7] review - 2 --- .../command/CommandAuthorizer.java | 5 +- .../command/CommandAuthorizerV1.java | 8 +- .../ql/parse/TestMaterializedCTEInputs.java | 124 ++++++++++++++++++ .../hive/ql/parse/TestSemanticAnalyzer.java | 57 -------- 4 files changed, 128 insertions(+), 66 deletions(-) create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java index 3b11a41e35f1..98f95fb37df1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java @@ -88,14 +88,13 @@ private static Set getInputs(BaseSemanticAnalyzer sem) { private static Set getOutputs(BaseSemanticAnalyzer sem) { Set additionalOutputs = new HashSet(); - for (WriteEntity e : sem.getAllOutputs()) { + for (WriteEntity e : sem.getOutputs()) { if (e.getType() == Entity.Type.PARTITION) { additionalOutputs.add(new WriteEntity(e.getTable(), e.getWriteType())); } } - // getAllOutputs() includes writes performed by materialized CTE sub-analyzers. // Sets.union keeps the values from the first set if they are present in both - return Sets.union(sem.getAllOutputs(), additionalOutputs); + return Sets.union(sem.getOutputs(), additionalOutputs); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java index d6629112e1f6..73630048532a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizerV1.java @@ -159,7 +159,7 @@ private static void authorizeInputs(HiveOperation op, BaseSemanticAnalyzer sem, } } - authorizeTable(op, sem, authorizer, tableUsePartLevelAuth, tab2Cols, tableAuthChecked, tbl); + authorizeTable(op, authorizer, tableUsePartLevelAuth, tab2Cols, tableAuthChecked, tbl); } } @@ -232,8 +232,7 @@ private static void getTablePartitionUsedColumns(HiveOperation op, BaseSemanticA } } - private static void authorizeTable(HiveOperation op, BaseSemanticAnalyzer sem, - HiveAuthorizationProvider authorizer, + private static void authorizeTable(HiveOperation op, HiveAuthorizationProvider authorizer, Map tableUsePartLevelAuth, Map> tab2Cols, Set tableAuthChecked, Table tbl) throws HiveException { // if we reach here, it means it needs to do a table authorization check, and the table authorization may @@ -241,9 +240,6 @@ private static void authorizeTable(HiveOperation op, BaseSemanticAnalyzer sem, if (tbl != null && !tableAuthChecked.contains(tbl.getTableName()) && !(Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName())))) { List cols = tab2Cols.get(tbl); - if ((cols == null || cols.isEmpty()) && sem.getColumnAccessInfo() != null) { - cols = sem.getColumnAccessInfo().getTableToColumnAccessMap().get(tbl.getCompleteName()); - } if (cols != null && cols.size() > 0) { authorizer.authorize(tbl, null, cols, op.getInputRequiredPrivileges(), null); } else { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java new file mode 100644 index 000000000000..3b51d04a6efe --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfForTest; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.database.drop.DropDatabaseDesc; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestMaterializedCTEInputs { + private static final String DB_NAME = "test_materialized_cte"; + private static final String TABLE_FQ_NAME = DB_NAME + "@src"; + + private static Hive db; + private static HiveConf conf; + + @BeforeAll + public static void beforeClass() throws Exception { + conf = new HiveConfForTest(TestMaterializedCTEInputs.class); + conf.set("hive.security.authorization.enabled", "false"); + conf.set("hive.security.authorization.manager", + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory"); + conf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 1); + conf.setBoolVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_FULL_AGGREGATE_ONLY, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); + db = Hive.get(conf); + SessionState.start(conf); + new DatabaseBuilder().setName(DB_NAME).create(db.getMSC(), conf); + SessionState.get().setCurrentDatabase(DB_NAME); + new TableBuilder().setDbName(DB_NAME).setTableName("src") + .addCol("key", "string") + .addCol("value", "string") + .addCol("col1", "int") + .create(db.getMSC(), conf); + } + + public static Stream casesForMaterializedCteInputs() { + return Stream.of( + Arguments.of("chain cte", "with q1 as ( select key from q2 where key = '5')," + + "q2 as ( select key from test_materialized_cte.src where key = '5') " + + "select * from (select key from q1) a", Set.of("key")), + Arguments.of("nested cte", "WITH q1 AS (" + + "WITH q2 AS (SELECT key, value FROM test_materialized_cte.src WHERE key = '4') " + + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " + + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key", Set.of("key", "value")) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("casesForMaterializedCteInputs") + void testMaterializedCteInputs(String type, String query, Set expectedCols) throws Exception { + HiveConf testConf = new HiveConf(conf); + Context ctx = new Context(testConf); + ASTNode astNode = ParseUtils.parse(query, ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); + SemanticAnalyzer analyzer = (SemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + analyzer.analyze(astNode, ctx); + + Set directInputs = analyzer.getInputs(); + Set allInputs = analyzer.getAllInputs(); + + assertTrue(directInputs.stream().noneMatch(e -> isTableNamed(e, "src")), + "Materialized CTE should not expose base table in direct inputs"); + assertTrue(allInputs.stream().anyMatch(e -> isTableNamed(e, "src")), + "Materialized CTE base table must appear in getAllInputs"); + + ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); + assertNotNull(columnAccessInfo); + List srcCols = columnAccessInfo.getTableToColumnAccessMap().get(TABLE_FQ_NAME); + assertNotNull(srcCols, "Column must include materialized CTE base table"); + assertTrue(new HashSet<>(srcCols).containsAll(expectedCols), + () -> "Expected columns " + expectedCols + " but got " + srcCols); + ctx.clear(); + } + + private static boolean isTableNamed(ReadEntity entity, String tableName) { + return entity.getTable() != null && tableName.equals(entity.getTable().getTableName()); + } + + @AfterAll + public static void afterClass() throws Exception { + try { + db.dropDatabase(new DropDatabaseDesc(DB_NAME, DB_NAME, true, true, true)); + } catch (Exception ignored) { + } + db.close(true); + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index d13eb7617436..17ac1b353bf2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -546,60 +545,4 @@ private void testMaterializeCTEUsesDDLFactory(boolean cboEnabled) throws Excepti cteAnalyzer[0] instanceof CreateTableAnalyzer); } } - - @Test - public void testMaterializedCteInputsAndColumnAccess() throws Exception { - createKeyValueTable("src"); - - HiveConf testConf = new HiveConf(conf); - testConf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 1); - testConf.setBoolVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_FULL_AGGREGATE_ONLY, false); - testConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); - - SessionState.start(testConf); - String[] queries = { - "with q1 as ( select key from q2 where key = '5')," - + "q2 as ( select key from src where key = '5') " - + "select * from (select key from q1) a", - "WITH q1 AS (" - + "WITH q2 AS (SELECT key, value FROM src WHERE key = '4') " - + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " - + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key" - }; - - SemanticAnalyzer[] analyzers = new SemanticAnalyzer[queries.length]; - for (int i = 0; i < queries.length; i++) { - Context ctx = new Context(testConf); - ASTNode astNode = ParseUtils.parse(queries[i], ctx); - QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); - SemanticAnalyzer analyzer = (SemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, astNode); - analyzer.initCtx(ctx); - analyzer.analyze(astNode, ctx); - analyzers[i] = analyzer; - } - - for (int i = 0; i < analyzers.length; i++) { - SemanticAnalyzer analyzer = analyzers[i]; - Set directInputs = analyzer.getInputs(); - Set allInputs = analyzer.getAllInputs(); - - assertTrue("Materialized CTE should not expose base table in direct inputs", - directInputs.stream().noneMatch(e -> isTableNamed(e, "src"))); - assertTrue("Nested materialized CTE base table must appear in getAllInputs", - allInputs.stream().anyMatch(e -> isTableNamed(e, "src"))); - - ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo(); - assertNotNull(columnAccessInfo); - List srcCols = columnAccessInfo.getTableToColumnAccessMap().get("default@src"); - assertNotNull("Column must include nested materialized CTE base table", srcCols); - assertTrue(srcCols.contains("key")); - if ( i == analyzers.length - 1) { - assertTrue(srcCols.contains("value")); - } - } - } - - private static boolean isTableNamed(ReadEntity entity, String tableName) { - return entity.getTable() != null && tableName.equals(entity.getTable().getTableName()); - } } From 3d01ffe93cc37003dfe1e157ceaf359bb94e49c1 Mon Sep 17 00:00:00 2001 From: zdeng Date: Wed, 26 Aug 2026 09:54:31 +0800 Subject: [PATCH 6/7] review - 3 --- .../hadoop/hive/ql/parse/ColumnAccessAnalyzer.java | 1 - .../apache/hadoop/hive/ql/parse/ColumnAccessInfo.java | 8 +++----- .../hadoop/hive/ql/parse/TestMaterializedCTEInputs.java | 9 +++++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java index 3200dca136a0..5a7f16b2f670 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java @@ -59,7 +59,6 @@ public ColumnAccessInfo analyzeColumnAccess(SemanticAnalyzer analyzer) throws Se } } } - // Every Analyzer holds its private rootClause columnAccessInfo.merge(getMaterializedCteColumnAccessInfo(analyzer.rootClause.asExecutionOrder())); return columnAccessInfo; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java index d06a06558d31..74f47b9481c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java @@ -93,16 +93,14 @@ public Map> getTableToColumnAllAccessMap() { } /** - * Merge direct column accesses from another ColumnAccessInfo into this one. + * Merge column accesses from another ColumnAccessInfo into this one. */ public void merge(ColumnAccessInfo other) { if (other == null) { return; } - for (Map.Entry> entry : other.getTableToColumnAccessMap().entrySet()) { - for (String col : entry.getValue()) { - add(entry.getKey(), col); - } + for (Map.Entry entry : other.tableToColumnAccessMap.entries()) { + tableToColumnAccessMap.put(entry.getKey(), entry.getValue()); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java index 3b51d04a6efe..6a888cb3131b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java @@ -77,7 +77,11 @@ public static Stream casesForMaterializedCteInputs() { Arguments.of("nested cte", "WITH q1 AS (" + "WITH q2 AS (SELECT key, value FROM test_materialized_cte.src WHERE key = '4') " + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " - + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key", Set.of("key", "value")) + + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key", Set.of("key", "value")), + Arguments.of("merge columns", "WITH q1 AS (" + + "WITH q2 AS (SELECT key FROM test_materialized_cte.src WHERE key = '4') " + + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) " + + "SELECT * FROM q1 t1 JOIN test_materialized_cte.src t2 ON t1.key = t2.key", Set.of("key", "value")) ); } @@ -92,11 +96,8 @@ void testMaterializedCteInputs(String type, String query, Set expectedCo analyzer.initCtx(ctx); analyzer.analyze(astNode, ctx); - Set directInputs = analyzer.getInputs(); Set allInputs = analyzer.getAllInputs(); - assertTrue(directInputs.stream().noneMatch(e -> isTableNamed(e, "src")), - "Materialized CTE should not expose base table in direct inputs"); assertTrue(allInputs.stream().anyMatch(e -> isTableNamed(e, "src")), "Materialized CTE base table must appear in getAllInputs"); From b95be171c9de702d9402ced049409eb156f4fd5f Mon Sep 17 00:00:00 2001 From: zdeng Date: Wed, 26 Aug 2026 15:28:40 +0800 Subject: [PATCH 7/7] review - 4 --- .../org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java index 74f47b9481c9..ea790218eeeb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java @@ -99,9 +99,7 @@ public void merge(ColumnAccessInfo other) { if (other == null) { return; } - for (Map.Entry entry : other.tableToColumnAccessMap.entries()) { - tableToColumnAccessMap.put(entry.getKey(), entry.getValue()); - } + tableToColumnAccessMap.putAll(other.tableToColumnAccessMap); } /**