From cecf57ecdc7d02f2488ea3a4a30f549f4f338169 Mon Sep 17 00:00:00 2001 From: ragupta Date: Sun, 1 Mar 2026 12:49:07 +0530 Subject: [PATCH] HIVE-29413:Avoid code duplication by updating getPartCols method for iceberg tables --- .../mapreduce/TestHCatMultiOutputFormat.java | 2 +- .../hive/ql/ddl/table/AlterTableUtils.java | 2 +- .../column/show/ShowColumnsOperation.java | 4 +- .../create/like/CreateTableLikeOperation.java | 2 +- .../table/info/desc/DescTableOperation.java | 2 +- .../formatter/TextDescTableFormatter.java | 4 +- .../JsonShowTableStatusFormatter.java | 2 +- .../TextShowTableStatusFormatter.java | 2 +- .../ddl/table/partition/PartitionUtils.java | 2 +- .../AlterTableExchangePartitionAnalyzer.java | 4 +- .../partition/show/ShowPartitionAnalyzer.java | 6 +-- .../archive/AlterTableArchiveOperation.java | 2 +- .../archive/AlterTableArchiveUtils.java | 2 +- .../archive/AlterTableUnarchiveOperation.java | 2 +- .../create/AbstractCreateViewAnalyzer.java | 2 +- .../hadoop/hive/ql/exec/ArchiveUtils.java | 6 +-- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 4 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../hive/ql/exec/repl/ReplLoadTask.java | 2 +- .../hive/ql/metadata/DummyPartition.java | 4 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 16 ++++---- .../HiveMaterializedViewsRegistry.java | 2 +- .../hadoop/hive/ql/metadata/Partition.java | 10 ++--- .../apache/hadoop/hive/ql/metadata/Table.java | 38 ++++++++++++++----- .../ql/optimizer/ColumnPrunerProcFactory.java | 3 +- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../ql/optimizer/ppr/PartitionPruner.java | 2 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- .../parse/ColumnStatsAutoGatherContext.java | 2 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 2 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 4 +- .../hive/ql/parse/MergeSemanticAnalyzer.java | 4 +- .../hadoop/hive/ql/parse/ParseUtils.java | 2 +- .../ql/parse/RewriteSemanticAnalyzer.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 6 +-- .../rewrite/CopyOnWriteMergeRewriter.java | 2 +- .../hive/ql/parse/rewrite/MergeRewriter.java | 4 +- .../ql/parse/rewrite/SplitMergeRewriter.java | 2 +- .../ql/parse/rewrite/SplitUpdateRewriter.java | 4 +- .../rewrite/sql/MultiInsertSqlGenerator.java | 6 +-- .../NativeAcidMultiInsertSqlGenerator.java | 6 +-- .../hive/ql/stats/ColStatsProcessor.java | 2 +- 46 files changed, 100 insertions(+), 89 deletions(-) diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index d87158b23fae..b67bc033290c 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -374,7 +374,7 @@ private List getTableData(String table, String database) throws Exceptio Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; - if (!tbl.getPartCols().isEmpty()) { + if (!tbl.getPartCols(true).isEmpty()) { List partitions = hive.getPartitions(tbl); List partDesc = new ArrayList(); List partLocs = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java index 17a964a44583..d3a7dbfe592a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java @@ -75,7 +75,7 @@ public static boolean isSchemaEvolutionEnabled(Table table, Configuration conf) } public static boolean isFullPartitionSpec(Table table, Map partitionSpec) { - for (FieldSchema partitionCol : table.getPartCols()) { + for (FieldSchema partitionCol : table.getPartCols(true)) { if (partitionSpec.get(partitionCol.getName()) == null) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java index 289479b7ee79..f37a7b175065 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java @@ -66,9 +66,7 @@ private List getColumnsByPattern() throws HiveException { private List getCols() throws HiveException { Table table = context.getDb().getTable(desc.getTableName()); - List allColumns = new ArrayList<>(); - allColumns.addAll(table.getCols()); - allColumns.addAll(table.getPartCols()); + List allColumns = new ArrayList<>(table.getAllCols()); return allColumns; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index 770724b90abf..021d32ec8681 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -100,7 +100,7 @@ private Table createViewLikeTable(Table oldTable) throws HiveException { setUserSpecifiedLocation(table); table.setFields(oldTable.getCols()); - table.setPartCols(oldTable.getPartCols()); + table.setPartCols(oldTable.getPartCols(true)); if (desc.getDefaultSerdeProps() != null) { for (Map.Entry e : desc.getDefaultSerdeProps().entrySet()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 6401f9a8d9b4..e9dd9a41b9f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -131,7 +131,7 @@ private void getColumnsNoColumnPath(Table table, Partition partition, List partitionColumns = null; // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation if (table.isPartitioned()) { - partitionColumns = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols(); + partitionColumns = table.getPartCols(true); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java index 073db26e756c..eafbecad6c21 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java @@ -69,7 +69,7 @@ private Map makeOneTableStatus(Table table, Hive db, HiveConf co builder.put("partitioned", table.isPartitioned()); if (table.isPartitioned()) { - builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(), + builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(true), Collections.emptyList())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java index 552dc310465b..550f3d72f937 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java @@ -73,7 +73,7 @@ private void writeStorageInfo(DataOutputStream out, Partition partition, Table t private void writeColumnsInfo(DataOutputStream out, Table table) throws IOException, UnsupportedEncodingException { String columns = MetaStoreUtils.getDDLFromFieldSchema("columns", table.getCols()); String partitionColumns = table.isPartitioned() ? - MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols()) : ""; + MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols(true)) : ""; out.write(Utilities.newLineCode); out.write(("columns:" + columns).getBytes(StandardCharsets.UTF_8)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index db7a5dfcd3d0..329c25fb64dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -150,7 +150,7 @@ public static List getPartitionsWithSpecs(Hive db, Table table, GetPa } private static String tablePartitionColNames(Table table) { - List partCols = table.getPartCols(); + List partCols = table.getPartCols(true); return String.join("/", partCols.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java index 6485627c7e6e..cf9be1bd78ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java @@ -84,7 +84,7 @@ protected void analyzeCommand(TableName tableName, Map partition if (AcidUtils.isTransactionalTable(sourceTable) || AcidUtils.isTransactionalTable(destTable)) { throw new SemanticException(ErrorMsg.EXCHANGE_PARTITION_NOT_ALLOWED_WITH_TRANSACTIONAL_TABLES.getMsg()); } - List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(), partitionSpecs); + List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(true), partitionSpecs); // check if source partition exists GetPartitionsFilterSpec sourcePartitionsFilterSpec = new GetPartitionsFilterSpec(); @@ -106,7 +106,7 @@ protected void analyzeCommand(TableName tableName, Map partition throw new SemanticException(ErrorMsg.PARTITION_VALUE_NOT_CONTINUOUS.getMsg(partitionSpecs.toString())); } - List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(), partitionSpecs); + List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(true), partitionSpecs); // check if dest partition exists GetPartitionsFilterSpec getDestPartitionsFilterSpec = new GetPartitionsFilterSpec(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java index c0bffcebdb23..4467f94af5da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java @@ -102,7 +102,7 @@ ExprNodeDesc getShowPartitionsFilter(Table table, ASTNode command) throws Semant if (astChild.getType() == HiveParser.TOK_WHERE) { RowResolver rwsch = new RowResolver(); Map colTypes = new HashMap(); - for (FieldSchema fs : table.getPartCols()) { + for (FieldSchema fs : table.getPartCols(true)) { rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.stringTypeInfo, null, true)); colTypes.put(fs.getName().toLowerCase(), fs.getType()); @@ -202,8 +202,8 @@ private String getShowPartitionsOrder(Table table, ASTNode command) throws Seman if (astChild.getType() == HiveParser.TOK_ORDERBY) { Map poses = new HashMap(); RowResolver rwsch = new RowResolver(); - for (int i = 0; i < table.getPartCols().size(); i++) { - FieldSchema fs = table.getPartCols().get(i); + for (int i = 0; i < table.getPartCols(true).size(); i++) { + FieldSchema fs = table.getPartCols(true).get(i); rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()), null, true)); poses.put(fs.getName().toLowerCase(), i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java index e218e590a24e..3af47970ec62 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java @@ -129,7 +129,7 @@ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitionColumns) throws SemanticException { - if (oldView.getPartCols().isEmpty() || oldView.getPartCols().equals(partitionColumns)) { + if (oldView.getPartCols(true).isEmpty() || oldView.getPartCols(true).equals(partitionColumns)) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index ebe8f2f52775..a4caa8c9f0bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -74,7 +74,7 @@ static public PartSpecInfo create(Table tbl, Map partSpec) // ARCHIVE PARTITION(hr='13') won't List prefixFields = new ArrayList(); List prefixValues = new ArrayList(); - List partCols = tbl.getPartCols(); + List partCols = tbl.getPartCols(true); Iterator itrPsKeys = partSpec.keySet().iterator(); for (FieldSchema fs : partCols) { if (!itrPsKeys.hasNext()) { @@ -222,7 +222,7 @@ public static int getArchivingLevel(Partition p) throws HiveException { * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { - List fields = p.getTable().getPartCols().subList(0, level); + List fields = p.getTable().getPartCols(true).subList(0, level); List values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); @@ -273,7 +273,7 @@ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, Map spec = new HashMap(partSpec); List reversedKeys = new ArrayList(); - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getPartCols(true)) { if (spec.containsKey(fs.getName())) { reversedKeys.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index a5bc66733f46..6e784424c0ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -294,7 +294,7 @@ public String getPartitionActualName(Partition pt) { */ private Map getPartitionColumnToPrimitiveCategory(Partition pt) { Map resultMap = new HashMap<>(); - for (FieldSchema schema: pt.getTable().getPartCols()) { + for (FieldSchema schema: pt.getTable().getPartCols(true)) { resultMap.put( schema.getName(), ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())).getPrimitiveCategory() @@ -976,7 +976,7 @@ private String getComment(Table table) { } private String getPartitionsForView(Table table) { - List partitionKeys = table.getPartCols(); + List partitionKeys = table.getPartCols(true); if (partitionKeys.isEmpty()) { return ""; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 3eca5531f127..eb273b3e8d9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -633,7 +633,7 @@ public void logMessage(LoadTableDesc tbd) { private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { - List partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); + List partVals = MetaStoreUtils.getPvals(table.getPartCols(true), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index f52251beb784..31220bf80d9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4315,7 +4315,7 @@ public static void setPartitionColumnNames(Configuration conf, TableScanOperator if (metadata == null) { return; } - List partCols = metadata.getPartCols(); + List partCols = metadata.getPartCols(true); if (partCols != null && !partCols.isEmpty()) { conf.set(serdeConstants.LIST_PARTITION_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(partCols)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 1e8a82b3a605..cb07ae6153cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -548,7 +548,7 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H } CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols(true)); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index c188eb09fdcf..2572c6a0c5a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -91,9 +91,7 @@ public List getValues() { values = new ArrayList<>(); // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation - for (FieldSchema fs : table.hasNonNativePartitionSupport() - ? table.getStorageHandler().getPartitionKeys(table) - : table.getPartCols()) { + for (FieldSchema fs : table.getPartCols(true)) { String val = partSpec.get(fs.getName()); values.add(val); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index b3977b8c9578..c776d79c3b77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -867,7 +867,7 @@ public void createTable(String tableName, List columns, List par FieldSchema part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key - tbl.getPartCols().add(part); + tbl.getPartCols(true).add(part); } } tbl.setSerializationLib(LazySimpleSerDe.class.getName()); @@ -1244,8 +1244,8 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio throws HiveException { try { Map newPartSpec = newPart.getSpec(); - if (oldPartSpec.keySet().size() != tbl.getPartCols().size() - || newPartSpec.keySet().size() != tbl.getPartCols().size()) { + if (oldPartSpec.keySet().size() != tbl.getPartCols(true).size() + || newPartSpec.keySet().size() != tbl.getPartCols(true).size()) { throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. "); } if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){ @@ -1253,7 +1253,7 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio } List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getPartCols(true)) { String val = oldPartSpec.get(field.getName()); if (val == null || val.length() == 0) { throw new HiveException("get partition: Value for key " @@ -3830,7 +3830,7 @@ public Partition getPartition(Table tbl, Map partSpec, boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException { tbl.validatePartColumnNames(partSpec, true); List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getPartCols(true)) { String val = partSpec.get(field.getName()); // enable dynamic partitioning if ((val == null && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING)) @@ -4219,7 +4219,7 @@ public List getPartitionNames(Table tbl, Map partSpec, s if (tbl.hasNonNativePartitionSupport()) { return tbl.getStorageHandler().getPartitionNames(tbl, partSpec); } - List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partSpec); + List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(true), partSpec); return getPartitionNamesByPartitionVals(tbl, pvals, max); } @@ -4461,7 +4461,7 @@ private List getPartitionsWithAuth(Table tbl, Map par throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } - List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partialPartSpec); + List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(true), partialPartSpec); List partitions = null; try { @@ -4770,7 +4770,7 @@ static List convertFromPartSpec(Iterator iterator, Tab || partitionWithoutSD.getRelativePath().isEmpty()) { if (tbl.getDataLocation() != null) { Path partPath = new Path(tbl.getDataLocation(), - Warehouse.makePartName(tbl.getPartCols(), + Warehouse.makePartName(tbl.getPartCols(true), partitionWithoutSD.getValues())); partitionLocation = partPath.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index f2379fb09f52..14869c60d801 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -409,7 +409,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema part_col : viewTable.getPartCols()) { + for (FieldSchema part_col : viewTable.getPartCols(true)) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 736e6e8c9f1a..f2261ad5748f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -125,7 +125,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getPartCols(true)) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " @@ -173,7 +173,7 @@ protected void initialize(Table table, // set default if location is not set and this is a physical // table partition (not a view partition) if (table.getDataLocation() != null) { - Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); + Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(true), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } @@ -200,7 +200,7 @@ protected void initialize(Table table, public String getName() { try { - return Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); + return Warehouse.makePartName(table.getPartCols(true), tPartition.getValues()); } catch (MetaException e) { throw new RuntimeException(e); } @@ -543,7 +543,7 @@ public void setLocation(String location) { public void setValues(Map partSpec) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : table.getPartCols()) { + for (FieldSchema field : table.getPartCols(true)) { String val = partSpec.get(field.getName()); if (val == null) { throw new HiveException( @@ -601,7 +601,7 @@ public Map, String> getSkewedColValueLocationMaps() { public void checkValidity() throws HiveException { if (!tPartition.getSd().equals(table.getSd())) { - Table.validateColumns(getCols(), table.getPartCols(), DDLUtils.isIcebergTable(table)); + Table.validateColumns(getCols(), table.getPartCols(true), DDLUtils.isIcebergTable(table)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 5be6735c1e3f..983219d4cd67 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -270,7 +270,7 @@ public void checkValidity(Configuration conf) throws HiveException { if (getCols().isEmpty()) { throw new HiveException("at least one column must be specified for the table"); } - validateColumns(getCols(), getPartCols(), DDLUtils.isIcebergTable(this)); + validateColumns(getCols(), getPartCols(false), DDLUtils.isIcebergTable(this)); if (!isView()) { if (null == getDeserializer(false)) { @@ -609,6 +609,18 @@ public boolean equals(Object obj) { return true; } + public List getPartCols(boolean needNonNativeParts) { + + List partKeys = getPartCols(); + + if (hasNonNativePartitionSupport() && needNonNativeParts) { + partKeys = getStorageHandler().getPartitionKeys(this); + } + + tTable.setPartitionKeys(partKeys); + return partKeys; + } + public List getPartCols() { List partKeys = tTable.getPartitionKeys(); if (partKeys == null) { @@ -619,15 +631,13 @@ public List getPartCols() { } public FieldSchema getPartColByName(String colName) { - return getPartCols().stream() + return getPartCols(true).stream() .filter(key -> key.getName().toLowerCase().equals(colName)) .findFirst().orElse(null); } public List getPartColNames() { - List partCols = hasNonNativePartitionSupport() ? - getStorageHandler().getPartitionKeys(this) : getPartCols(); - return partCols.stream().map(FieldSchema::getName) + return getPartCols(true).stream().map(FieldSchema::getName) .collect(Collectors.toList()); } @@ -777,8 +787,16 @@ private List getColsInternal(boolean forMs) { */ public List getAllCols() { ArrayList f_list = new ArrayList(); - f_list.addAll(getCols()); - f_list.addAll(getPartCols()); + Map colNameWiseMap = new HashMap<>(); + for (FieldSchema f : getCols()) { + colNameWiseMap.put(f.getName(), f); + f_list.add(f); + } + for (FieldSchema f : getPartCols(true)) { + if (!colNameWiseMap.containsKey(f.getName())) { + f_list.add(f); + } + } return f_list; } @@ -828,7 +846,7 @@ public void setOutputFormatClass(String name) throws HiveException { public boolean isPartitioned() { return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : - CollectionUtils.isNotEmpty(getPartCols()); + CollectionUtils.isNotEmpty(getPartCols(true)); } public void setFields(List fields) { @@ -1026,7 +1044,7 @@ public boolean isMaterializedView() { public LinkedHashMap createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { - List fsl = getPartCols(); + List fsl = getPartCols(true); List tpl = tp.getValues(); LinkedHashMap spec = new LinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { @@ -1168,7 +1186,7 @@ public static void validateColumns(List columns, List } colNames.add(colName); } - if (partCols != null) { + if (partCols != null && !icebergTable) { // there is no overlap between columns and partitioning columns for (FieldSchema partCol: partCols) { String colName = normalize(partCol.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 3d3e4ce7663f..ec0096b96ea9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -807,8 +807,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, for (FieldNode col : cols) { int index = originalOutputColumnNames.indexOf(col.getFieldName()); Table tab = cppCtx.getParseContext().getViewProjectToTableSchema().get(op); - List fullFieldList = new ArrayList(tab.getCols()); - fullFieldList.addAll(tab.getPartCols()); + List fullFieldList = new ArrayList(tab.getAllCols()); cppCtx.getParseContext().getColumnAccessInfo() .add(tab.getCompleteName(), fullFieldList.get(index).getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 5af9ff1af4cf..209b7e8ecb7f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -2152,7 +2152,7 @@ static void usePartitionColumns(Properties properties, Table table, List if (properties.containsKey(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS)) { usePartitionColumns(properties, partColNames); } else { - List partCols = table.getPartCols(); + List partCols = table.getPartCols(true); String partNames = partCols.stream().map(FieldSchema::getName).collect(Collectors.joining("/")); String partTypes = partCols.stream().map(FieldSchema::getType).collect(Collectors.joining(":")); properties.setProperty( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index b1bc9eaf0a75..b18ffbd8c653 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -525,7 +525,7 @@ static private boolean pruneBySequentialScan(Table tab, List partitio } private static List extractPartColTypes(Table tab) { - List pCols = tab.getPartCols(); + List pCols = tab.getPartCols(true); List partColTypeInfos = new ArrayList<>(pCols.size()); for (FieldSchema pCol : pCols) { partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 06912a1b3226..29dcef31005d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,7 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); + newTable.getPartCols(true), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; rewrittenCtx.setIsUpdateDeleteMerge(false); //it's set in parseRewrittenQuery() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index a0a0f7500806..bfa67a673e08 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1200,7 +1200,7 @@ public TableSpec(Table tableHandle, List partitions) { if (partitions != null && !partitions.isEmpty()) { this.specType = SpecType.STATIC_PARTITION; this.partitions = partitions; - List partCols = this.tableHandle.getPartCols(); + List partCols = this.tableHandle.getPartCols(true); this.partSpec = new LinkedHashMap<>(); for (FieldSchema partCol : partCols) { partSpec.put(partCol.getName(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index f1e9da25ebec..a2a95c2e428b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3058,7 +3058,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema part_col : tabMetaData.getPartCols()) { + for (FieldSchema part_col : tabMetaData.getPartCols(false)) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 1b6f73ea264f..3c1ce35563c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -83,7 +83,7 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.isInsertInto = isInsertInto; this.origCtx = ctx; columns = tbl.getCols(); - partitionColumns = tbl.getPartCols(); + partitionColumns = tbl.getPartCols(false); } public List getLoadFileWork() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 023934d9eb24..324011d7a132 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -278,7 +278,7 @@ private static String genRewrittenQuery(Table tbl, List colNames, List existingTablePartCols = table.getPartCols(); + List existingTablePartCols = table.getPartCols(true); List importedTablePartCols = tableDesc.getPartCols(); if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index eb4a73f1e5e9..182d66475dab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -511,7 +511,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc // Partition spec was already validated by caller when create TableSpec object. // So, need not validate inpPartSpec here. - List parts = table.getPartCols(); + List parts = table.getPartCols(true); if (tableTree.getChildCount() >= 2) { ASTNode partSpecNode = (ASTNode) tableTree.getChild(1); inpPartSpec = new HashMap<>(partSpecNode.getChildCount()); @@ -561,7 +561,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc } rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0)))); - addPartitionColsToInsert(table.getPartCols(), inpPartSpec, rewrittenQueryStr); + addPartitionColsToInsert(table.getPartCols(true), inpPartSpec, rewrittenQueryStr); rewrittenQueryStr.append(" select * from "); rewrittenQueryStr.append(tempTblName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 882840ffef5a..77ca9e66a85a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -230,7 +230,7 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause String deleteExtraPredicate) throws SemanticException { assert whenMatchedUpdateClause.getType() == HiveParser.TOK_MATCHED; assert getWhenClauseOperation(whenMatchedUpdateClause).getType() == HiveParser.TOK_UPDATE; - Map newValuesMap = new HashMap<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + Map newValuesMap = new HashMap<>(targetTable.getAllCols().size()); ASTNode setClause = (ASTNode)getWhenClauseOperation(whenMatchedUpdateClause).getChild(0); //columns being updated -> update expressions; "setRCols" (last param) is null because we use actual expressions //before re-parsing, i.e. they are known to SemanticAnalyzer logic @@ -432,7 +432,7 @@ private static final class OnClauseAnalyzer { HiveConf conf, String onClauseAsString) { this.onClause = onClause; allTargetTableColumns.addAll(targetTable.getCols()); - allTargetTableColumns.addAll(targetTable.getPartCols()); + allTargetTableColumns.addAll(targetTable.getPartCols(true)); this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery); this.conf = conf; this.onClauseAsString = onClauseAsString; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 9964b9369065..b2cacfa882da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -692,7 +692,7 @@ public static Map> getFullPartitionSpecs( */ private static int calculatePartPrefix(Table tbl, Set partSpecKeys) { int partPrefixToDrop = 0; - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getPartCols(true)) { if (!partSpecKeys.contains(fs.getName())) { break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java index 101f6b1fc3d8..a70667129cc1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java @@ -119,7 +119,7 @@ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) thr String columnName = normalizeColName(colName.getText()); // Make sure this isn't one of the partitioning columns, that's not supported. - for (FieldSchema fschema : targetTable.getPartCols()) { + for (FieldSchema fschema : targetTable.getPartCols(true)) { if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f08808b01415..705d74dcb8ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2206,7 +2206,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } else { // partition spec is not specified but column schema can have partitions specified - for(FieldSchema f : targetTable.getPartCols()) { + for(FieldSchema f : targetTable.getPartCols(true)) { //parser only allows foo(a,b), not foo(foo.a, foo.b) targetColumns.remove(f.getName()); } @@ -12035,7 +12035,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns - for (FieldSchema part_col : tab.getPartCols()) { + for (FieldSchema part_col : tab.getPartCols(true)) { LOG.trace("Adding partition col: " + part_col); rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); @@ -12301,7 +12301,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String if (tab.isPartitioned() && !tab.hasNonNativePartitionSupport()) { List cols = new ArrayList(); if (qbp.getAnalyzeRewrite() != null) { - List partitionCols = tab.getPartCols(); + List partitionCols = tab.getPartCols(true); for (FieldSchema fs : partitionCols) { cols.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java index d43adacf0f7c..94499932a68a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java @@ -195,7 +195,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(hintStr); hintStr = null; } - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); values.addAll(sqlGenerator.getDeleteValues(Context.Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index 44b0a0d9e5a6..ee179926f9a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -211,7 +211,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(" -- update clause").append("\n"); List valuesAndAcidSortKeys = new ArrayList<>( - targetTable.getCols().size() + targetTable.getPartCols().size() + 1); + targetTable.getAllCols().size() + 1); valuesAndAcidSortKeys.addAll(sqlGenerator.getSortKeys(Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), valuesAndAcidSortKeys); sqlGenerator.appendInsertBranch(hintStr, valuesAndAcidSortKeys); @@ -237,7 +237,7 @@ protected void addValues(Table targetTable, String targetAlias, Map values.add( + targetTable.getPartCols(true).forEach(fieldSchema -> values.add( formatter.apply(fieldSchema.getName()))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java index 5ba8bd0193e9..e68b827da27d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java @@ -55,7 +55,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau String onClauseAsString = mergeStatement.getOnClauseAsText(); sqlGenerator.append(" -- update clause (insert part)\n"); - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); sqlGenerator.appendInsertBranch(hintStr, values); hintStr = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java index 16724f42d70a..899b1dd4e919 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java @@ -96,8 +96,8 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB insertValues.add(sqlGenerator.qualify(identifier)); } - if (updateBlock.getTargetTable().getPartCols() != null) { - updateBlock.getTargetTable().getPartCols().forEach( + if (updateBlock.getTargetTable().getPartCols(true) != null) { + updateBlock.getTargetTable().getPartCols(true).forEach( fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 7587daf13055..246930b75358 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -111,7 +111,7 @@ public void appendPartitionColsOfTarget() { */ public void appendPartitionCols(Table table) { // If the table is partitioned we have to put the partition() clause in - List partCols = table.getPartCols(); + List partCols = table.getPartCols(true); if (partCols == null || partCols.isEmpty()) { return; } @@ -148,11 +148,11 @@ public void removeLastChar() { } public void appendPartColsOfTargetTableWithComma(String alias) { - if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) { + if (targetTable.getPartCols(true) == null || targetTable.getPartCols(true).isEmpty()) { return; } queryStr.append(','); - appendCols(targetTable.getPartCols(), alias, null, FieldSchema::getName); + appendCols(targetTable.getPartCols(true), alias, null, FieldSchema::getName); } public void appendAllColsOfTargetTable(String prefix) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java index 87e426800442..9692d08b362e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java @@ -36,7 +36,7 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName @Override public void appendAcidSelectColumns(Operation operation) { queryStr.append("ROW__ID,"); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols(true)) { String identifier = HiveUtils.unparseIdentifier(fieldSchema.getName(), this.conf); queryStr.append(identifier); queryStr.append(","); @@ -45,9 +45,9 @@ public void appendAcidSelectColumns(Operation operation) { @Override public List getDeleteValues(Operation operation) { - List deleteValues = new ArrayList<>(1 + targetTable.getPartCols().size()); + List deleteValues = new ArrayList<>(1 + targetTable.getPartCols(true).size()); deleteValues.add(qualify("ROW__ID")); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols(true)) { deleteValues.add(qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))); } return deleteValues; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..a3d7be721985 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -156,7 +156,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List