diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6268efa4e50a..a6a73eee20b6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2130,6 +2130,15 @@ public static enum ConfVars { "Whether to use codec pool in ORC. Disable if there are bugs with codec reuse."), HIVE_ICEBERG_STATS_SOURCE("hive.iceberg.stats.source", "iceberg", "Use stats from iceberg table snapshot for query planning. This has two values metastore and iceberg"), + HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL("hive.iceberg.stats.collect.partlevel", false, + "Whether column statistics of a partitioned Iceberg table are collected per partition.\n" + + "Per partition statistics let the planner estimate from the partitions a query scans, but\n" + + "only an ANALYZE statement writes them. INSERT and CTAS maintain the table level ones.\n" + + "After a change, existing statistics of the other granularity are ignored until recomputed."), + HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK("hive.iceberg.stats.max.snapshot.lookback", 20, + "How many snapshots a read of per partition column statistics walks back through to tell\n" + + "which partitions the writes since have changed. Each one costs reading the manifests it\n" + + "wrote, so a file further back than this cannot be judged and is not served."), HIVE_ICEBERG_EXPIRE_SNAPSHOT_NUMTHREADS("hive.iceberg.expire.snapshot.numthreads", 4, "The number of threads to be used for deleting files during expire snapshot. If set to 0 or below it uses the" + " default DirectExecutorService"), diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index d652dbc6ef51..dcd44b6033ab 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -48,6 +47,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.SQLViewRepresentation; @@ -119,17 +119,18 @@ public static void alterTable( } } - public static List getPartitionKeys(org.apache.iceberg.Table table, int specId) { - Schema schema = table.specs().get(specId).schema(); - List hiveSchema = HiveSchemaUtil.convert(schema); - Map colNameToColType = hiveSchema.stream() - .collect(Collectors.toMap(FieldSchema::getName, FieldSchema::getType)); - return table.specs().get(specId).fields().stream() - .map(partField -> new FieldSchema( - schema.findColumnName(partField.sourceId()), - colNameToColType.get(schema.findColumnName(partField.sourceId())), - String.format("Transform: %s", partField.transform().toString())) - ) + public static List getPartitionKeys(org.apache.iceberg.Table table) { + Schema schema = table.spec().schema(); + + return table.spec().fields().stream() + .map(partField -> { + Types.NestedField col = schema.findField(partField.sourceId()); + return new FieldSchema( + col.name().toLowerCase(), // HMS lowercases column names + HiveSchemaUtil.convertToTypeString(col.type()), + "Transform: %s".formatted(partField.transform()) + ); + }) .toList(); } @@ -143,7 +144,7 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co result.setTableType(TableType.EXTERNAL_TABLE.toString()); // TODO: Revert after HIVE-29633 is fixed - // result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); + // result.setPartitionKeys(getPartitionKeys(table)); result.setPartitionKeys(Lists.newArrayList()); TableMetadata metadata = ((BaseTable) table).operations().current(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java index 2a8701379086..f66f098654b7 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java @@ -505,9 +505,9 @@ private void commitTable(FileIO io, ExecutorService executor, OutputTable output .orElse(RewritePolicy.DEFAULT.name())); if (rewritePolicy != RewritePolicy.DEFAULT) { - String partitionPath = jobContexts.stream() + String partitionName = jobContexts.stream() .findAny() - .map(x -> x.getJobConf().get(IcebergCompactionService.PARTITION_PATH)) + .map(x -> x.getJobConf().get(IcebergCompactionService.PARTITION_NAME)) .orElse(null); long fileSizeThreshold = jobContexts.stream() @@ -516,7 +516,7 @@ private void commitTable(FileIO io, ExecutorService executor, OutputTable output .map(Long::parseLong) .orElse(-1L); - commitCompaction(table, snapshotId, startTime, filesForCommit, partitionPath, fileSizeThreshold); + commitCompaction(table, snapshotId, startTime, filesForCommit, partitionName, fileSizeThreshold); } else { commitOverwrite(table, branchName, snapshotId, startTime, filesForCommit); } @@ -623,14 +623,14 @@ private void commit(Transaction txn, SnapshotUpdate update) { * @param snapshotId The snapshot id of the table to use for validation * @param startTime The start time of the commit - used only for logging * @param results The object containing the new files - * @param partitionPath The path of the compacted partition + * @param partitionName The path of the compacted partition */ private void commitCompaction(Table table, Long snapshotId, long startTime, FilesForCommit results, - String partitionPath, long fileSizeThreshold) { + String partitionName, long fileSizeThreshold) { List existingDataFiles = - IcebergCompactionUtil.getDataFiles(table, snapshotId, partitionPath, fileSizeThreshold); + IcebergCompactionUtil.getDataFiles(table, snapshotId, partitionName, fileSizeThreshold); List existingDeleteFiles = fileSizeThreshold == -1 ? - IcebergCompactionUtil.getDeleteFiles(table, snapshotId, partitionPath) : Collections.emptyList(); + IcebergCompactionUtil.getDeleteFiles(table, snapshotId, partitionName) : Collections.emptyList(); Transaction txn = IcebergAcidUtil.getOrCreateTransaction(table, jobConf); @@ -644,7 +644,7 @@ private void commitCompaction(Table table, Long snapshotId, long startTime, File } commit(txn, rewriteFiles); LOG.info("Compaction commit took {} ms for table: {} partition: {} with {} file(s)", - System.currentTimeMillis() - startTime, table, StringUtils.defaultString(partitionPath, "N/A"), + System.currentTimeMillis() - startTime, table, StringUtils.defaultString(partitionName, "N/A"), results.dataFiles().size()); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java index 83d00942edd5..68dc85413742 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java @@ -140,8 +140,7 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - this.projectedSchema = - projectedSchema(conf, serDeProperties.getProperty(Catalogs.NAME), tableSchema, jobConf); + this.projectedSchema = projectedSchema(conf, serDeProperties, tableSchema, jobConf); if (!IcebergTableUtil.isFanoutEnabled(serDeProperties::getProperty)) { // ClusteredWriter requires that records are ordered by partition keys. @@ -156,9 +155,11 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - private static Schema projectedSchema(Configuration conf, String tableName, Schema tableSchema, - Map jobConf) { + private static Schema projectedSchema(Configuration conf, Properties serDeProperties, + Schema tableSchema, Map jobConf) { + String tableName = serDeProperties.getProperty(Catalogs.NAME); Context.Operation operation = HiveCustomStorageHandlerUtils.getWriteOperation(conf::get, tableName); + if (operation == null) { jobConf.put(InputFormatConfig.CASE_SENSITIVE, "false"); String[] selectedColumns = ColumnProjectionUtils.getReadColumnNames(conf); @@ -180,11 +181,15 @@ private static Schema projectedSchema(Configuration conf, String tableName, Sche } boolean isCOW = IcebergTableUtil.isCopyOnWriteMode(operation, conf::get); if (isCOW) { - return getSchemaWithRowLineage(IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()), conf); + return getSchemaWithRowLineage( + IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), false), conf); } switch (operation) { case DELETE: - return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()); + boolean isMergeTask = HiveCustomStorageHandlerUtils.isMergeTaskEnabled( + key -> serDeProperties.getProperty(key, conf.get(key)), + tableName); + return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), isMergeTask); case UPDATE: return IcebergAcidUtil.createSerdeSchemaForUpdate(tableSchema.columns()); case OTHER: diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b23c20991856..b46179c8dfb9 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -20,15 +20,14 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.io.Serializable; import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -39,24 +38,22 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.SnapshotContext; import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -64,10 +61,10 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -142,8 +139,6 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.FindFiles; -import org.apache.iceberg.GenericBlobMetadata; -import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionData; @@ -188,12 +183,10 @@ import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.hive.actions.HiveIcebergDeleteOrphanFiles; import org.apache.iceberg.mr.hive.plan.IcebergBucketFunction; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsReader; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter; import org.apache.iceberg.mr.hive.udf.GenericUDFIcebergZorder; -import org.apache.iceberg.puffin.Blob; import org.apache.iceberg.puffin.BlobMetadata; -import org.apache.iceberg.puffin.Puffin; -import org.apache.iceberg.puffin.PuffinCompressionCodec; -import org.apache.iceberg.puffin.PuffinWriter; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; @@ -203,9 +196,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.SerializationUtil; import org.apache.iceberg.util.SnapshotUtil; import org.slf4j.Logger; @@ -214,7 +205,7 @@ import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.FILE_PATH; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_HASH; -import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_PROJECTION; +import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_NAME; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_SPEC_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_LINEAGE_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_POSITION; @@ -236,15 +227,14 @@ public class HiveIcebergStorageHandler extends DefaultStorageHandler implements private static final String TABLE_NAME_SEPARATOR = ".."; public static final String TABLE_DEFAULT_LOCATION = "TABLE_DEFAULT_LOCATION"; - private static final String PARTITION = "partition"; + private static final String PARTITION = IcebergTableUtil.PARTITION_FIELD; private static final String PARTITION_STATS_PREFIX = "partitionStats."; - public static final String STATS = "/stats/snap-"; public static final String COPY_ON_WRITE = RowLevelOperationMode.COPY_ON_WRITE.modeName(); public static final String MERGE_ON_READ = RowLevelOperationMode.MERGE_ON_READ.modeName(); private static final List ACID_VIRTUAL_COLS = ImmutableList.of( - PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION, PARTITION_PROJECTION); + PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION); private static final List ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA = schema(ACID_VIRTUAL_COLS); @@ -427,7 +417,8 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese List subExprNodes = pushedPredicate.getChildren(); Set skipList = - Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) + Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, PARTITION_NAME, + ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) .map(VirtualColumn::getName).collect(Collectors.toSet()); if (subExprNodes.removeIf(nodeDesc -> nodeDesc.getCols() != null && @@ -447,7 +438,6 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese return predicate; } - @Override public boolean canProvideBasicStatistics() { return true; @@ -520,7 +510,9 @@ private Map getBasicStatistics(org.apache.hadoop.hive.ql.metadat if (snapshot == null) { stats = emptyStatsMap(); - } else if (!HiveMetaHook.ICEBERG.equals(getStatsSource()) && !quickStats) { + } else if (!HiveMetaHook.ICEBERG.equals(getStatsSource()) && !quickStats && + hmsTable.getSnapshotRef() == null) { + // the metastore parameters describe the table, not a branch: use the snapshot's counters stats = hmsTable.getParameters(); } else { @@ -627,7 +619,7 @@ public Map computeBasicStatistics(org.apache.hadoop.hive.ql.meta .commit(); statsTable = tx.table(); } - statsFile = PartitionStatsHandler.computeAndWriteStatsFile(statsTable); + statsFile = PartitionStatsHandler.computeAndWriteStatsFile(statsTable, snapshot.snapshotId()); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -718,108 +710,39 @@ private Table getTable(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { return IcebergTableUtil.getTable(conf, hmsTable.getTTable(), skipCache); } + @Override + public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, boolean partitionLevel) { + // asked of the configuration alone: a CREATE has to ask before the table it describes exists, + // and whether that table has partitions at all is settled by the caller + return canSetColStatistics(hmsTable) && + partitionLevel == HiveConf.getBoolVar(conf, ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL); + } + @Override public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { return HiveMetaHook.ICEBERG.equals(getStatsSource()); } @Override - public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colStats) { - Table tbl = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return writeColStats(colStats, tbl); + public boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, String colName) { + if (canSetColStatistics(hmsTable)) { + return IcebergTableUtil.colStatsAccurate(hmsTable, conf); + } + // the metastore holds them, and its single row describes the table: a branch has none + return hmsTable.getSnapshotRef() == null && + StatsSetupConst.areColumnStatsUptoDate(hmsTable.getParameters(), colName); } - @SuppressWarnings("checkstyle:CyclomaticComplexity") - private boolean writeColStats(List colStats, Table tbl) { - try { - if (!shouldRewriteColStats(tbl)) { - checkAndMergeColStats(colStats, tbl); - } - StatisticsFile statisticsFile; - String statsPath = tbl.location() + STATS + UUID.randomUUID(); - - try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) - .createdBy(Constants.HIVE_ENGINE) - .build()) { - - long snapshotId = tbl.currentSnapshot().snapshotId(); - long snapshotSequenceNumber = tbl.currentSnapshot().sequenceNumber(); - Schema schema = tbl.spec().schema(); - - boolean first = true; - - for (ColumnStatistics stats : colStats) { - boolean isTblLevel = stats.getStatsDesc().isIsTblLevel(); - - Map properties = isTblLevel ? Map.of() : - Map.of(PARTITION, String.valueOf(stats.getStatsDesc().getPartName())); - - List statsObjects = isTblLevel ? - stats.getStatsObj() : List.of(stats); - - List fieldIds = null; - - if (!isTblLevel) { - // For partition-level stats, we emit one blob per partition; - // therefore, only the first blob should contain the actual fieldIds. - fieldIds = !first ? List.of(-1) : - stats.getStatsObj().stream() - .map(obj -> schema.findField(obj.getColName()).fieldId()) - .toList(); - first = false; - } - - for (Serializable statsObj : statsObjects) { - byte[] serialized = SerializationUtils.serialize(statsObj); - - if (isTblLevel) { - fieldIds = List.of(schema.findField( - ((ColumnStatisticsObj) statsObj).getColName()).fieldId()); - } - - writer.add(new Blob( - ColumnStatisticsObj.class.getSimpleName(), - fieldIds, - snapshotId, - snapshotSequenceNumber, - ByteBuffer.wrap(serialized), - PuffinCompressionCodec.NONE, - properties - )); - } - } - - writer.finish(); - - statisticsFile = - new GenericStatisticsFile( - snapshotId, - statsPath, - writer.fileSize(), - writer.footerSize(), - writer.writtenBlobsMetadata().stream() - .map(GenericBlobMetadata::from) - .collect(ImmutableList.toImmutableList()) - ); - } catch (IOException e) { - LOG.warn("Unable to write column stats to the Puffin file: {}", e.getMessage()); - - Path path = new Path(statsPath); - FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - fs.delete(path, false); - } - return false; - } - tbl.updateStatistics() - .setStatistics(statisticsFile) - .commit(); - return true; - - } catch (Exception e) { - LOG.warn("Unable to invalidate or merge column stats: {}", e.getMessage()); + @Override + public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, + Iterator colStats) { + Table tbl = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); + // a write to a branch moves that branch's head, leaving the table's current snapshot behind + Snapshot snapshot = IcebergTableUtil.getTableSnapshot(tbl, hmsTable); + if (snapshot == null || !colStats.hasNext()) { + return false; } - return false; + return IcebergColStatsWriter.writeColStats(tbl, snapshot, colStats, conf); } @Override @@ -827,15 +750,12 @@ public boolean canProvideColStatistics(org.apache.hadoop.hive.ql.metadata.Table Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); if (snapshot != null) { - return canSetColStatistics(hmsTable) && canProvideColStats(table, snapshot.snapshotId()); + return canSetColStatistics(hmsTable) && + IcebergTableUtil.findColStatsFile(table, snapshot.snapshotId(), conf) != null; } return false; } - private boolean canProvideColStats(Table table, long snapshotId) { - return IcebergTableUtil.getColStatsPath(table, snapshotId) != null; - } - @Override public List getColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colNames) { @@ -852,6 +772,11 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta return Lists.newArrayList(); } + // this returns the whole-table statistics, so judge the whole-table file: a write since it was + // written leaves them stale, and the metastore withholds its own on the same terms + if (!IcebergTableUtil.colStatsAccurate(table, snapshot, false)) { + return Lists.newArrayList(); + } Predicate filter; if (colNames != null) { Set columns = Sets.newHashSet(colNames); @@ -863,7 +788,7 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta filter = null; } - return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); + return IcebergColStatsReader.readColStats(table, snapshot.snapshotId(), filter); } @Override @@ -880,13 +805,28 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION); double ndvTuner = MetastoreConf.getDoubleVar(getConf(), MetastoreConf.ConfVars.STATS_NDV_TUNER); + // per partition blobs are what is read below, so judge the per partition file + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(table, snapshot.snapshotId(), true); + if (statsFile == null) { + return new AggrStats(Collections.emptyList(), 0); + } Set partitions = Sets.newHashSet(partNames); - Predicate filter = metadata -> partitions.contains(metadata.properties().get(PARTITION)); - - List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); - - partStats.forEach(colStats -> - colStats.getStatsObj().removeIf(statsObj -> !colNames.contains(statsObj.getColName()))); + // a partition written since the file was written is no longer described by it + Predicate upToDate = + IcebergTableUtil.upToDateColStats(table, snapshot, statsFile, conf, true); + Map> statsByPart = IcebergColStatsReader.readPartColStats(table, statsFile, + partition -> partitions.contains(partition) && upToDate.test(partition), Sets.newHashSet(colNames)); + + List partStats = Lists.newArrayList(); + statsByPart.forEach((partition, statsObjs) -> { + // the metastore counts a partition as found only when it has every column asked about + if (statsObjs.size() == colNames.size()) { + ColumnStatisticsDesc statsDesc = + new ColumnStatisticsDesc(false, hmsTable.getDbName(), hmsTable.getTableName()); + statsDesc.setPartName(partition); + partStats.add(new ColumnStatistics(statsDesc, statsObjs)); + } + }); List colStatsList = MetaStoreServerUtils.aggrPartitionStats(partStats, MetaStoreUtils.getDefaultCatalog(conf), hmsTable.getDbName(), hmsTable.getTableName(), @@ -902,7 +842,7 @@ public Long getRowCount(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { if (hmsTable.getMetaTable() != null) { return null; } - return getStatsSource().equals(HiveMetaHook.ICEBERG) ? + return getStatsSource().equals(HiveMetaHook.ICEBERG) || hmsTable.getSnapshotRef() != null ? snapshotRowCount(hmsTable) : metastoreRowCount(hmsTable); } @@ -954,48 +894,6 @@ private String getStatsSource() { .toUpperCase(); } - private boolean shouldRewriteColStats(Table tbl) { - return SessionStateUtil.getQueryState(conf) - .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) - .orElse(false) || - IcebergTableUtil.getColStatsPath(tbl) != null; - } - - private void checkAndMergeColStats(List statsNew, Table tbl) throws InvalidObjectException { - Long previousSnapshotId = tbl.currentSnapshot().parentId(); - if (previousSnapshotId != null && canProvideColStats(tbl, previousSnapshotId)) { - - boolean isTblLevel = statsNew.getFirst().getStatsDesc().isIsTblLevel(); - Map oldStatsMap = Maps.newHashMap(); - - List statsOld = IcebergTableUtil.readColStats(tbl, previousSnapshotId, null); - - if (!isTblLevel) { - for (ColumnStatistics statsObjOld : (List) statsOld) { - oldStatsMap.put(statsObjOld.getStatsDesc().getPartName(), statsObjOld); - } - } else { - statsOld = Collections.singletonList( - new ColumnStatistics(null, (List) statsOld)); - } - for (ColumnStatistics statsObjNew : statsNew) { - String partitionKey = statsObjNew.getStatsDesc().getPartName(); - ColumnStatistics statsObjOld = isTblLevel ? - (ColumnStatistics) statsOld.getFirst() : oldStatsMap.get(partitionKey); - - if (statsObjOld != null && statsObjOld.getStatsObjSize() != 0 && !statsObjNew.getStatsObj().isEmpty()) { - MetaStoreServerUtils.mergeColStats(statsObjNew, statsObjOld); - if (!isTblLevel) { - oldStatsMap.remove(partitionKey); - } - } - } - if (!isTblLevel) { - statsNew.addAll(oldStatsMap.values()); - } - } - } - /** * Iceberg's optimistic concurrency control fails to provide means for IOW and Insert operations isolation. * Use `hive.txn.ext.locking.enabled` config to create Hive locks in order to guarantee data consistency. @@ -1043,22 +941,10 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m } @Override - public Map> getPartitionTransformSpecs( - org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) { - return Collections.emptyMap(); - } - Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return table.specs().entrySet().stream().flatMap(e -> - e.getValue().fields().stream() - .filter(f -> !f.transform().isVoid()) - .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); - spec.setFieldName(f.name()); - return Pair.of(e.getKey(), spec); - })) - .collect(Collectors.groupingBy( - Pair::first, Collectors.mapping(Pair::second, Collectors.toList()))); + public Function partitionNameResolver( + org.apache.hadoop.hive.ql.metadata.Table hmsTable, StructObjectInspector inspector) { + return IcebergTableUtil.partitionNameFunction( + IcebergTableUtil.getTable(conf, hmsTable.getTTable()), inspector); } private List getWriteSortTransformSpecs(Table table) { @@ -1290,7 +1176,6 @@ public void storageHandlerCommit(Properties commitProperties, Operation operatio } } - @Override public HiveIcebergOutputCommitter getOutputCommitter() { return new HiveIcebergOutputCommitter(); @@ -2249,9 +2134,7 @@ public void validatePartSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Objects.requireNonNull(field, String.format("%s is not a partition column", spec.getKey())); // If the partition spec value is null, it's a dynamic partition column. if (spec.getValue() != null) { - Object partKeyVal = Conversions.fromPartitionString(field.type(), spec.getValue()); - Objects.requireNonNull(partKeyVal, - String.format("Partition spec value for column : %s is invalid", field.name())); + IcebergTableUtil.parsePartitionValue(field.type(), spec.getValue()); } } } @@ -2462,7 +2345,7 @@ public List getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Tab return hmsTable.getPartitionKeys(); } Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return MetastoreUtil.getPartitionKeys(icebergTable, icebergTable.spec().specId()); + return MetastoreUtil.getPartitionKeys(icebergTable); } @Override @@ -2489,7 +2372,6 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta } Set partitions = Sets.newHashSet(); - String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULT_PARTITION_NAME); try (CloseableIterable tasks = scan.planFiles()) { FluentIterable.from(tasks) @@ -2499,10 +2381,8 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta PartitionData partitionData = IcebergTableUtil.toPartitionData(task.partition(), spec.partitionType()); String partName = IcebergTableUtil.toPartitionName(spec, partitionData); - Map partSpecMap = - IcebergTableUtil.makeSpecFromName(partName, spec, partitionData, defaultPartitionName); - - DummyPartition partition = new DummyPartition(hmsTable, partName, partSpecMap); + DummyPartition partition = + new DummyPartition(hmsTable, partName, IcebergTableUtil.specFromName(partName)); partitions.add(partition); }); } catch (IOException e) { @@ -2572,7 +2452,6 @@ public boolean supportsDefaultColumnValues(Map tblProps) { return IcebergTableUtil.formatVersion(tblProps) >= 3; } - private static List schema(List exprs) { return exprs.stream().map(v -> new FieldSchema(v.getName(), v.getTypeInfo().getTypeName(), "")) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java index c615c1d82c29..b881e07b2eb8 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java @@ -25,15 +25,15 @@ import java.util.Objects; import java.util.Optional; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.io.PositionDeleteInfo; import org.apache.hadoop.hive.ql.io.RowLineageInfo; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.iceberg.ContentFile; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; @@ -51,16 +51,13 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.SerializationUtil; -import org.apache.iceberg.util.StructProjection; public class IcebergAcidUtil { private IcebergAcidUtil() { } - private static final Types.NestedField PARTITION_STRUCT_META_COL = null; // placeholder value in the map private static final Map FILE_READ_META_COLS = Maps.newLinkedHashMap(); - private static final Map VIRTUAL_COLS_TO_META_COLS = Maps.newLinkedHashMap(); public static final String META_TABLE_PROPERTY = "metaTable"; private static final Map DELETE_FILE_META_COLS = Maps.newLinkedHashMap(); public static final Integer PARTITION_PROJECTION_COLUMN_ID = Integer.MAX_VALUE - 6; @@ -71,59 +68,52 @@ private IcebergAcidUtil() { DELETE_FILE_META_COLS.put(MetadataColumns.ROW_POSITION, 1); FILE_READ_META_COLS.put(MetadataColumns.SPEC_ID, 0); - FILE_READ_META_COLS.put(PARTITION_STRUCT_META_COL, 1); - FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 2); - FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_SPEC_ID.getName(), MetadataColumns.SPEC_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_HASH.getName(), PARTITION_STRUCT_META_COL); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.FILE_PATH.getName(), MetadataColumns.FILE_PATH); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_POSITION.getName(), MetadataColumns.ROW_POSITION); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_LINEAGE_ID.getName(), MetadataColumns.ROW_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER.getName(), - MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER); + FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 1); + FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 2); } private static final Types.NestedField PARTITION_HASH_META_COL = Types.NestedField.required( MetadataColumns.PARTITION_COLUMN_ID, MetadataColumns.PARTITION_COLUMN_NAME, Types.LongType.get()); - public static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( + private static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( PARTITION_PROJECTION_COLUMN_ID, PARTITION_PROJECTION_COLUMN_NAME, Types.StringType.get()); + private static final Map SERDE_META_COLS = Maps.newLinkedHashMap(); + // a merge task reads delete files, so its writer has no row data to derive the partition key from + private static final Map MERGE_SERDE_META_COLS = Maps.newLinkedHashMap(); + static { SERDE_META_COLS.put(MetadataColumns.SPEC_ID, 0); SERDE_META_COLS.put(PARTITION_HASH_META_COL, 1); SERDE_META_COLS.put(MetadataColumns.FILE_PATH, 2); SERDE_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - SERDE_META_COLS.put(PARTITION_PROJECTION, 4); + + MERGE_SERDE_META_COLS.putAll(SERDE_META_COLS); + MERGE_SERDE_META_COLS.put(PARTITION_PROJECTION, 4); } /** * @param dataCols The columns of the original file read schema - * @param table The table object - it is used for populating the partition struct meta column * @return The schema for reading files, extended with metadata columns */ - public static Schema createFileReadSchemaWithVirtualColums(List dataCols, Table table) { + public static Schema createFileReadSchemaWithVirtualColums(List dataCols) { List cols = Lists.newArrayListWithCapacity(dataCols.size() + FILE_READ_META_COLS.size()); - FILE_READ_META_COLS.forEach((metaCol, index) -> { - if (metaCol == PARTITION_STRUCT_META_COL) { - cols.add(MetadataColumns.metadataColumn(table, MetadataColumns.PARTITION_COLUMN_NAME)); - } else { - cols.add(metaCol); - } - }); + FILE_READ_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); cols.addAll(dataCols); return new Schema(cols); } /** * @param dataCols The columns of the serde projection schema + * @param isMergeTask Whether the schema is for a merge task, which also carries the partition key * @return The schema for SerDe operations, extended with metadata columns needed for deletes */ - public static Schema createSerdeSchemaForDelete(List dataCols) { - List cols = Lists.newArrayListWithCapacity(dataCols.size() + SERDE_META_COLS.size()); - SERDE_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); + public static Schema createSerdeSchemaForDelete(List dataCols, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; + List cols = Lists.newArrayListWithCapacity(dataCols.size() + metaCols.size()); + cols.addAll(metaCols.keySet()); cols.addAll(dataCols); return new Schema(cols); } @@ -133,14 +123,17 @@ public static Schema createSerdeSchemaForDelete(List dataCols * the field values from `rec`. * @param rec The record read by the file scan task, which contains both the metadata fields and the row data fields * @param rowData The record object to populate with the rowData fields only + * @param isMergeTask Whether the record was built by a merge task * @return The position delete object */ - public static PositionDelete getPositionDelete(Record rec, Record rowData) { + public static PositionDelete getPositionDelete(Record rec, Record rowData, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; PositionDelete positionDelete = PositionDelete.create(); - String filePath = rec.get(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - Long filePosition = rec.get(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); + String filePath = rec.get(metaCols.get(MetadataColumns.FILE_PATH), String.class); + Long filePosition = rec.get(metaCols.get(MetadataColumns.ROW_POSITION), Long.class); - int dataOffset = SERDE_META_COLS.size(); // position in the rec where the actual row data begins + int dataOffset = metaCols.size(); // position in the rec where the actual row data begins for (int i = dataOffset; i < rec.size(); ++i) { rowData.set(i - dataOffset, rec.get(i)); } @@ -169,14 +162,8 @@ public static int parseSpecId(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.SPEC_ID), Integer.class); } - public static long computePartitionHash(Record rec) { - StructProjection part = rec.get(FILE_READ_META_COLS.get(PARTITION_STRUCT_META_COL), StructProjection.class); - // we need to compute a hash value for the partition struct so that it can be used as a sorting key - return computeHash(part); - } - public static PartitionKey parsePartitionKey(Record rec) { - String serializedStr = rec.get(SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); + String serializedStr = rec.get(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); return SerializationUtil.deserializeFromBase64(serializedStr); } @@ -190,15 +177,11 @@ public static String getSerializedPartitionKey(StructLike structLike, PartitionS return SerializationUtil.serializeToBase64(partitionKey); } - public static String parseFilePath(Record rec) { - return rec.get(FILE_READ_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - } - public static String getFilePath(Record rec) { return rec.get(DELETE_FILE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); } - public static long parseFilePosition(Record rec) { + public static long getFilePosition(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); } @@ -261,12 +244,23 @@ public static class VirtualColumnAwareIterator implements CloseableIterator currentIterator, Schema expectedSchema, Configuration conf) { + private final int specId; + private final long partitionHash; + private final String filePath; + + public VirtualColumnAwareIterator(CloseableIterator currentIterator, List columns, + Configuration conf, FileScanTask task) { this.currentIterator = currentIterator; this.current = GenericRecord.create( - new Schema(expectedSchema.columns().subList(4, expectedSchema.columns().size()))); + new Schema(columns.subList(FILE_READ_META_COLS.size(), columns.size()))); this.conf = conf; + + this.specId = task.file().specId(); + this.partitionHash = computeHash(task.file().partition()); + this.filePath = task.file().location(); + + IOContextMap.get(conf).setPartitionName( + IcebergTableUtil.toPartitionName(task.spec(), task.file().partition())); } @Override @@ -285,11 +279,10 @@ public T next() { GenericRecord rec = (GenericRecord) next; IcebergAcidUtil.copyFields(rec, FILE_READ_META_COLS.size(), current.size(), current); PositionDeleteInfo.setIntoConf(conf, - IcebergAcidUtil.parseSpecId(rec), - IcebergAcidUtil.computePartitionHash(rec), - IcebergAcidUtil.parseFilePath(rec), - IcebergAcidUtil.parseFilePosition(rec), - StringUtils.EMPTY); + specId, + partitionHash, + filePath, + IcebergAcidUtil.getFilePosition(rec)); RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec), RowLineageReader.readLastUpdatedSequenceNumber(rec), conf); return (T) current; @@ -299,17 +292,20 @@ public T next() { public static class MergeTaskVirtualColumnAwareIterator implements CloseableIterator { private final CloseableIterator currentIterator; - private final GenericRecordBuilder recordBuilder; - private final PartitionSpec partitionSpec; - private final StructLike partition; + private final MergeTaskRecordBuilder recordBuilder; - public MergeTaskVirtualColumnAwareIterator( - CloseableIterator currentIterator, Schema expectedSchema, ContentFile contentFile, Table table) { + private final int specId; + private final long partitionHash; + private final String serializedPartitionKey; + + public MergeTaskVirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, + PartitionSpec spec, ContentFile file) { this.currentIterator = currentIterator; - this.partition = contentFile.partition(); - this.recordBuilder = new GenericRecordBuilder<>( - new Schema(expectedSchema.columns().subList(0, expectedSchema.columns().size()))); - this.partitionSpec = table.specs().get(contentFile.specId()); + this.recordBuilder = new MergeTaskRecordBuilder<>(expectedSchema); + + this.specId = spec.specId(); + this.partitionHash = computeHash(file.partition()); + this.serializedPartitionKey = getSerializedPartitionKey(file.partition(), spec); } @Override @@ -326,44 +322,45 @@ public boolean hasNext() { public T next() { T next = currentIterator.next(); GenericRecord rec = (GenericRecord) next; - return recordBuilder.withSpecId(partitionSpec.specId()) - .withPartitionHash(computeHash(partition)) + + return recordBuilder.withSpecId(specId) + .withPartitionHash(partitionHash) .withFilePath(IcebergAcidUtil.getFilePath(rec)) .withFilePosition(IcebergAcidUtil.getDeleteFilePosition(rec)) - .withPartitionKey(getSerializedPartitionKey(partition, partitionSpec)).build(); + .withPartitionKey(serializedPartitionKey) + .build(); } } - private static final class GenericRecordBuilder { - + private static final class MergeTaskRecordBuilder { private final GenericRecord current; - GenericRecordBuilder(Schema schema) { + MergeTaskRecordBuilder(Schema schema) { current = GenericRecord.create(schema); } - public GenericRecordBuilder withSpecId(int specId) { - current.set(SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); + public MergeTaskRecordBuilder withSpecId(int specId) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); return this; } - public GenericRecordBuilder withPartitionHash(long partitionHash) { - current.set(SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); + public MergeTaskRecordBuilder withPartitionHash(long partitionHash) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); return this; } - public GenericRecordBuilder withFilePath(String filePath) { - current.set(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); + public MergeTaskRecordBuilder withFilePath(String filePath) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); return this; } - public GenericRecordBuilder withFilePosition(long filePosition) { - current.set(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); + public MergeTaskRecordBuilder withFilePosition(long filePosition) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); return this; } - public GenericRecordBuilder withPartitionKey(String serializedPartitionKey) { - current.set(SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); + public MergeTaskRecordBuilder withPartitionKey(String serializedPartitionKey) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); return this; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 897771bbb3cc..b9ce6710ce90 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -21,15 +21,12 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -41,16 +38,16 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; -import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.TableFetcher; @@ -68,10 +65,13 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.util.Sets; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataOperations; import org.apache.iceberg.DeleteFiles; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ManageSnapshots; @@ -82,6 +82,7 @@ import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatistics; import org.apache.iceberg.PartitionStatisticsFile; @@ -89,7 +90,9 @@ import org.apache.iceberg.PartitionsTable; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; @@ -98,6 +101,9 @@ import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdatePartitionSpec; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.InternalRecordWrapper; +import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -106,9 +112,9 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.puffin.BlobMetadata; -import org.apache.iceberg.puffin.Puffin; -import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.mr.hive.serde.objectinspector.IcebergObjectInspector; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -118,8 +124,6 @@ import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.ByteBuffers; -import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PartitionUtil; import org.apache.iceberg.util.SnapshotUtil; import org.apache.iceberg.util.StructProjection; @@ -142,6 +146,9 @@ public class IcebergTableUtil { private static final String SPEC_ID_FIELD = "spec_id"; private static final String NULL_VALUE = "null"; + public static final String PARTITION_FIELD = "partition"; + private static final String MODIFIED_PARTITIONS_PREFIX = "modifiedPartitions."; + private IcebergTableUtil() { } @@ -253,18 +260,85 @@ static Snapshot getTableSnapshot(Table table, String snapshotRef) { return table.currentSnapshot(); } - static String getColStatsPath(Table table) { - return getColStatsPath(table, table.currentSnapshot().snapshotId()); + /** + * The newest column statistics file describing the snapshot: its own, or - across commits that + * rewrite files without changing any rows (compaction) - an ancestor's. Of the granularity the + * table maintains, which is the one every write produces. + */ + static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { + return getColStatsFile(table, snapshotId, isPartitionStats(table, conf)); + } + + public static StatisticsFile getColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + // A rewrite leaves every row in the partition it was already in, and so separates statistics + // from nothing - unless the table has evolved, where compaction selects the rows of the older + // specs and writes them under the current one. That is the same condition the compactor + // branches on, and only per partition statistics can tell the difference. + boolean rewritesKeepPartitions = !partitionLevel || table.specs().size() == 1; + return colStatsFileOf( + table, snapshotId, partitionLevel, + snapshot -> !rewritesKeepPartitions || !DataOperations.REPLACE.equals(snapshot.operation())); + } + + /** + * The nearest column statistics file describing the snapshot: its own, or the closest + * ancestor's. Statistics of an ancestor describe an earlier state of the data, which + * {@link #colStatsAccurate} reports and the planner treats as partial. + */ + static StatisticsFile findColStatsFile(Table table, long snapshotId, Configuration conf) { + return findColStatsFile(table, snapshotId, isPartitionStats(table, conf)); + } + + public static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + // a snapshot holding no rows (truncate) ends the walk: what precedes it didn't survive + return colStatsFileOf( + table, snapshotId, partitionLevel, IcebergTableUtil::isEmptySnapshot); + } + + /** The statistics file of the snapshot or of an ancestor, up to the one {@code last} names. */ + private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boolean partitionLevel, + Predicate last) { + if (table.statisticsFiles().isEmpty()) { + return null; + } + for (Snapshot snapshot = table.snapshot(snapshotId); snapshot != null; + snapshot = snapshot.parentId() != null ? table.snapshot(snapshot.parentId()) : null) { + StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), partitionLevel); + if (statsFile != null) { + return statsFile; + } + if (last.test(snapshot)) { + return null; + } + } + return null; } - static String getColStatsPath(Table table, long snapshotId) { + /** + * The file whose blobs are Hive's own - Iceberg keeps statistics of its own in the same format - + * at the asked-for granularity: a blob describing one partition names it in its metadata. + */ + private static StatisticsFile lookupColStatsFile(Table table, long snapshotId, boolean partitionLevel) { return table.statisticsFiles().stream() - .filter(stats -> stats.snapshotId() == snapshotId) - .filter(stats -> stats.blobMetadata().stream() - .anyMatch(metadata -> ColumnStatisticsObj.class.getSimpleName().equals(metadata.type())) - ) - .map(StatisticsFile::path) - .findAny().orElse(null); + .filter(stats -> stats.snapshotId() == snapshotId) + .filter(stats -> stats.blobMetadata().stream().anyMatch(partitionLevel ? + metadata -> IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + metadata.properties().containsKey(PARTITION_FIELD) : + metadata -> (IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) || + IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(metadata.type())) && + !metadata.properties().containsKey(PARTITION_FIELD))) + .findAny().orElse(null); + } + + /** Whether the snapshot holds no rows. */ + public static boolean isEmptySnapshot(Snapshot snapshot) { + return snapshot != null && snapshot.summary() != null && + NumberUtils.toLong(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP), -1) == 0; + } + + /** Whether the table's column statistics are maintained per partition, as the flag directs. */ + public static boolean isPartitionStats(Table table, Configuration conf) { + return HiveConf.getBoolVar(conf, ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL) && table.spec().isPartitioned(); } static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotId) { @@ -281,9 +355,7 @@ static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotI * @return iceberg partition spec, always non-null */ public static PartitionSpec spec(Configuration configuration, Schema schema) { - List partitionBy = SessionStateUtil - .getResource(configuration, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) - .map(o -> (List) o).orElse(null); + List partitionBy = TransformSpec.fromQueryState(configuration); if (partitionBy == null) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); @@ -458,23 +530,6 @@ public static void performMetadataDelete(Table icebergTable, String branchName, deleteFiles.deleteFromRowFilter(exp).commit(); } - /** - * Parses an Iceberg partition path into a Hive-compatible spec map, representing null partition - * values with the Hive default partition name. - */ - public static Map makeSpecFromName(String partName, PartitionSpec spec, PartitionData data, - String defaultPartitionName) { - Map partSpecMap = Maps.newLinkedHashMap(); - Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); - - List fields = spec.fields(); - for (int i = 0; i < fields.size(); i++) { - if (data.get(i) == null) { - partSpecMap.put(fields.get(i).name(), defaultPartitionName); - } - } - return partSpecMap; - } public static PartitionData toPartitionData(StructLike key, Types.StructType keyType) { PartitionData keyTemplate = new PartitionData(keyType); @@ -493,8 +548,62 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy * name an unpartitioned spec renders. Statistics and partition pruning join on this name, so both must * render it the same way. */ - static String toPartitionName(PartitionSpec spec, PartitionData data) { - return StringUtils.defaultIfEmpty(spec.partitionToPath(data), DummyPartition.VOID); + @SuppressWarnings("unchecked") + public static String toPartitionName(PartitionSpec spec, StructLike data) { + String path = spec.partitionToPath(data); + // an unpartitioned spec renders nothing: its rows belong to the table-level partition + if (path.isEmpty()) { + return DummyPartition.VOID; + } + // a field of no value renders as the text "null", which a value of that text renders as too, + // so the partition of no value takes the name Hive keeps for it and the two stay two + List fields = spec.fields(); + Class[] javaClasses = spec.javaClasses(); + String[] parts = path.split("/"); + + boolean renamed = false; + for (int i = 0; i < fields.size() && i < parts.length; i++) { + if (data.get(i, (Class) javaClasses[i]) == null) { + parts[i] = fields.get(i).name() + "=" + ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal; + renamed = true; + } + } + return renamed ? String.join("/", parts) : path; + } + + /** + * Returns a function naming the partition a row belongs to, where the row holds no more than the + * columns the spec partitions on. The spec derives the tuple itself, so a name matches the one a + * write gives the files it puts in that partition. Deriving one costs the same as writing a row, + * so the function holds on to what that takes and is worth reusing over rows an inspector reads. + */ + public static Function partitionNameFunction(Table table, StructObjectInspector inspector) { + // the deserializer reads the row by position, so the schema has to follow the inspector rather + // than the table, and a name reaches here lowercased however the table spells it + Schema sourceSchema = new Schema(inspector.getAllStructFieldRefs().stream() + .map(field -> Preconditions.checkNotNull(table.schema().caseInsensitiveFindField(field.getFieldName()), + "Table %s partitions on %s, which it has no column for", table.name(), field.getFieldName())) + .toList()); + Deserializer deserializer = new Deserializer.Builder() + .schema(sourceSchema) + .sourceInspector(inspector) + .writerInspector((StructObjectInspector) IcebergObjectInspector.create(sourceSchema)) + .build(); + + // the whole schema, not the row's columns: PartitionKey needs an accessor for every field of + // the spec, including a void one, whose source column the row does not carry + GenericRecord record = GenericRecord.create(table.schema()); + InternalRecordWrapper wrapper = new InternalRecordWrapper(table.schema().asStruct()); + PartitionKey partitionKey = new PartitionKey(table.spec(), table.schema()); + + return row -> { + // a row whose every value is null is indistinguishable from no row at all once serialized + Record source = row == null ? null : deserializer.deserialize(row); + sourceSchema.columns().forEach(column -> + record.setField(column.name(), source == null ? null : source.getField(column.name()))); + partitionKey.partition(wrapper.wrap(record)); + return toPartitionName(table.spec(), partitionKey); + }; } /** @@ -571,6 +680,13 @@ private static Expression buildExpression(Map partitionSpec, return predicate; } + static Object parsePartitionValue(Type type, String value) { + boolean isNullValue = NULL_VALUE.equalsIgnoreCase(value) && + !Types.StringType.get().equals(type); + return isNullValue ? + null : Conversions.fromPartitionString(type, value); + } + private static Expression buildFieldPredicate(Types.StructType partitionType, String fieldName, String value, String keyPrefix) { String key = keyPrefix + fieldName; @@ -621,13 +737,6 @@ private static Expression buildTransformedFieldPredicate( return Expressions.equal(partitionKey, transformedValue); } - private static Object parsePartitionValue(Type type, String value) { - boolean isNullValue = NULL_VALUE.equalsIgnoreCase(value) && - !Types.StringType.get().equals(type); - return isNullValue ? - null : Conversions.fromPartitionString(type, value); - } - /** * Returns a partition matching the given partition spec. * With partition evolution, multiple partitions may match; returns the one from the highest spec ID. @@ -719,14 +828,12 @@ private static List getPartitionNames(Configuration conf, .filter(evaluator::eval) .transform(row -> { PartitionSpec spec = icebergTable.specs().get(row.get(SPEC_IDX, Integer.class)); - if (!spec.isPartitioned()) { - return null; - } + // rows written before the table was partitioned: toPartitionName gives them the + // synthetic partition name PartitionData data = toPartitionData( row.get(PART_IDX, StructProjection.class), partitionType, spec.partitionType()); return Maps.immutableEntry(toPartitionName(spec, data), spec.specId()); }) - .filter(Objects::nonNull) .toSortedList(specIdComparator).stream() .map(Map.Entry::getKey) .toList(); @@ -802,34 +909,123 @@ public static TransformSpec getTransformSpec(Table table, String transformName, return spec; } - public static List readColStats(Table table, Long snapshotId, Predicate filter) { - List colStats = Lists.newArrayList(); - String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); - if (statsPath == null) { - LOG.warn("Column stats file not found for snapshot: {}", snapshotId); - return colStats; + /** + * Whether the stored column statistics still describe the snapshot the table names: its current + * one, or a branch's head, which a write to that branch moves on its own. + */ + public static boolean colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Configuration conf) { + Table table = getTable(conf, hmsTable.getTTable()); + Snapshot snapshot = getTableSnapshot(table, hmsTable); + return snapshot != null && colStatsAccurate(table, snapshot, conf); + } + + /** + * Whether the stored column statistics still describe the table: the current snapshot owns them, + * or only row-preserving commits (compaction) separate it from the snapshot that does. Derived + * from the table metadata, so it holds for the writes of every engine. + */ + public static boolean colStatsAccurate(Table table, Snapshot snapshot, Configuration conf) { + return colStatsAccurate(table, snapshot, isPartitionStats(table, conf)); + } + + /** The same, judging the granularity the caller goes on to read. */ + static boolean colStatsAccurate(Table table, Snapshot snapshot, boolean partitionLevel) { + return getColStatsFile(table, snapshot.snapshotId(), partitionLevel) != null; + } + + /** + * The partitions written since a snapshot. Null when the walk cannot answer - an expired + * snapshot broke the chain, or the bound was reached - and every partition then counts as + * changed. The bound is on manifests read, not on snapshots walked. + */ + static Set partitionsChangedSince(Table table, Snapshot snapshot, long sinceSnapshotId, + Configuration conf, boolean capped) { + // the walk reads manifests, and every column of a query asks the same question of them + String cacheKey = MODIFIED_PARTITIONS_PREFIX + table.name() + '.' + snapshot.snapshotId() + '.' + + sinceSnapshotId + '.' + capped; + Optional cached = SessionStateUtil.getResource(conf, cacheKey); + if (cached.isPresent()) { + return ((Optional>) cached.get()).orElse(null); } - try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)).build()) { - List blobMetadata = reader.fileMetadata().blobs(); + Set changed = walkPartitionsChangedSince(table, snapshot, sinceSnapshotId, conf, capped); + SessionStateUtil.addResource(conf, cacheKey, Optional.ofNullable(changed)); + return changed; + } - if (filter != null) { - blobMetadata = blobMetadata.stream().filter(filter) - .toList(); + private static Set walkPartitionsChangedSince(Table table, Snapshot snapshot, + long sinceSnapshotId, Configuration conf, boolean capped) { + // the bound is what a read will wait for; a write settles its file for good, so it walks the + // whole way + int lookback = capped ? + HiveConf.getIntVar(conf, ConfVars.HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK) : Integer.MAX_VALUE; + Set changed = Sets.newHashSet(); + boolean rewritesKeepPartitions = table.specs().size() == 1; + Snapshot current = snapshot; + + for (int read = 0; current != null && current.snapshotId() != sinceSnapshotId; ) { + // A rewrite (REPLACE) leaves every row in the partition it was already in, unless the table + // has evolved: compaction then selects the rows of the older specs and writes them under the + // current one. An overwrite of partitions commits OVERWRITE, despite the name of its API. + if (!rewritesKeepPartitions || !DataOperations.REPLACE.equals(current.operation())) { + // only the snapshots whose manifests are read cost anything, so only they are counted + if (read == lookback) { + LOG.info("Stopped after {} snapshots of {}; the statistics written at snapshot {} " + + "cannot be judged", lookback, table.name(), sinceSnapshotId); + return null; + } + if (!collectChangedPartitions(table, current, changed)) { + return null; + } + read++; } - Iterator it = Iterables.transform(reader.readAll(blobMetadata), Pair::second).iterator(); - LOG.info("Using column stats from: {}", statsPath); + Long parentId = current.parentId(); + current = parentId != null ? table.snapshot(parentId) : null; + } + // the file has to sit on the history walked, or what happened in between is unknown + return current == null ? null : changed; + } - while (it.hasNext()) { - byte[] byteBuffer = ByteBuffers.toByteArray(it.next()); - colStats.add(SerializationUtils.deserialize(byteBuffer)); + /** + * Records which partitions a snapshot changed, naming each file under its own spec. False when + * a delete of no partition is reached, since it applies to the rows of every one and names none. + * Stored + * entries are named the same way: an ANALYZE names each group after the file its rows came from, + * and a write only ever lands in a partition of the spec current when it ran. + */ + private static boolean collectChangedPartitions(Table table, Snapshot snapshot, Set changed) { + for (ContentFile file : changedFiles(table, snapshot)) { + PartitionSpec spec = table.specs().get(file.specId()); + // an equality delete of no partition applies to the rows of every one, and names none of them + if (file.content() != FileContent.DATA && !spec.isPartitioned() && table.spec().isPartitioned()) { + return false; } - } catch (Exception e) { - LOG.warn("Unable to read column stats: {}", e.getMessage()); + changed.add(toPartitionName(spec, file.partition())); } - return colStats; + return true; } + /** Every file a snapshot added or removed, whether it holds rows or deletes them. */ + private static Iterable> changedFiles(Table table, Snapshot snapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(snapshot).build(); + return Iterables.concat( + changes.addedDataFiles(), changes.removedDataFiles(), + changes.addedDeleteFiles(), changes.removedDeleteFiles()); + } + + /** + * Whether a stored entry still describes its partition. A file describes the snapshot it was + * written for, so only what happened after it matters. False for all of them when the writes in + * between cannot be traced. + */ + public static Predicate upToDateColStats(Table table, Snapshot snapshot, + StatisticsFile statsFile, Configuration conf, boolean capped) { + Set changed = + partitionsChangedSince(table, snapshot, statsFile.snapshotId(), conf, capped); + return partition -> changed != null && !changed.contains(partition); + } + + public static ExecutorService newDeleteThreadPool(String completeName, int numThreads) { AtomicInteger deleteThreadsIndex = new AtomicInteger(0); return Executors.newFixedThreadPool(numThreads, runnable -> { @@ -963,7 +1159,8 @@ public static > Set getPartitionNames(Table ice int tableSpecId = icebergTable.spec().specId(); for (T file : files) { if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { - String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); + String partName = toPartitionName( + icebergTable.specs().get(file.specId()), file.partition()); partitions.add(partName); } } @@ -974,13 +1171,21 @@ public static List convertNameToMetastorePartition(org.apache.hadoop. Collection partNames) { List partitions = Lists.newArrayList(); for (String partName : partNames) { - Map partSpecMap = Maps.newLinkedHashMap(); - Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); - partitions.add(new DummyPartition(hmsTable, partName, partSpecMap)); + partitions.add(new DummyPartition(hmsTable, partName, specFromName(partName))); } return partitions; } + /** + * The values a partition name encodes. The name is one this table rendered, so it always parses, + * which the overload returning the map does not get to assume. + */ + static Map specFromName(String partName) { + Map partSpecMap = Maps.newLinkedHashMap(); + Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); + return partSpecMap; + } + public static TableFetcher getTableFetcher(IMetaStoreClient msc, String catalogName, String dbPattern, String tablePattern) { return new TableFetcher.Builder(msc, catalogName, dbPattern, tablePattern).tableTypes( diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java index 18a3622fa185..20649f580681 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; public class IcebergCompactionService extends CompactionService { - public static final String PARTITION_PATH = "compaction_partition_path"; + public static final String PARTITION_NAME = "compaction_partition_name"; private static final String CLASS_NAME = IcebergCompactionService.class.getName(); private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); @@ -76,7 +76,9 @@ public Boolean compact(Table table, CompactionInfo ci) throws Exception { try { CompactorPipeline compactorPipeline = compactorFactory.getCompactorPipeline(table, conf, ci, msc); - computeStats = collectGenericStats; + // no follow-up ANALYZE: basic statistics live in the table metadata, column statistics are + // gathered by the compaction query itself, and a compaction commit outdates neither + computeStats = false; LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + ", id:" + ci.id + " with compute stats set to " + computeStats); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java index 46bf6269f759..fad31bc9ed4d 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java @@ -29,6 +29,7 @@ import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.hive.IcebergTableUtil; import org.apache.iceberg.relocated.com.google.common.collect.Lists; public class IcebergCompactionUtil { @@ -40,43 +41,44 @@ private IcebergCompactionUtil() { /** * This method implements a common filter that is used in several places in Iceberg compaction code. * Its aim is to determine if the provided file needs to be handled when compacting a partition whose path is equal to - * the provided partitionPath. Returns true when one of the following conditions is true, otherwise returns false: + * the provided partitionName. Returns true when one of the following conditions is true, otherwise returns false: * 1. table is unpartitioned - * 2. partitionPath is null and the file belongs to the non-latest partition spec - * 3. partitionPath is not null and the file belongs to the partition whose path is the partitionPath + * 2. partitionName is null and the file belongs to the non-latest partition spec + * 3. partitionName is not null and the file belongs to the partition of that name * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name * @param file Data or Delete file */ - public static boolean shouldIncludeForCompaction(Table table, String partitionPath, ContentFile file) { + public static boolean shouldIncludeForCompaction(Table table, String partitionName, ContentFile file) { return !table.spec().isPartitioned() || - partitionPath == null && file.specId() != table.spec().specId() || - partitionPath != null && - table.specs().get(file.specId()).partitionToPath(file.partition()).equals(partitionPath); + partitionName == null && file.specId() != table.spec().specId() || + partitionName != null && + IcebergTableUtil.toPartitionName(table.specs().get(file.specId()), file.partition()) + .equals(partitionName); } - public static boolean shouldIncludeForCompaction(Table table, String partitionPath, ContentFile file, + public static boolean shouldIncludeForCompaction(Table table, String partitionName, ContentFile file, long fileSizeThreshold) { - return shouldIncludeForCompaction(table, partitionPath, file) && + return shouldIncludeForCompaction(table, partitionName, file) && (fileSizeThreshold == -1 || file.fileSizeInBytes() < fileSizeThreshold); } /** * Returns table's list of data files as following: * 1. If the table is unpartitioned, returns all data files. - * 2. If partitionPath is not provided, returns all data files that belong to the non-latest partition spec. - * 3. If partitionPath is provided, returns all data files that belong to the corresponding partition. + * 2. If partitionName is not provided, returns all data files that belong to the non-latest partition spec. + * 3. If partitionName is provided, returns all data files that belong to the corresponding partition. * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name */ - public static List getDataFiles(Table table, Long snapshotId, String partitionPath, + public static List getDataFiles(Table table, Long snapshotId, String partitionName, long fileSizeThreshold) { CloseableIterable scanTasks = table.newBatchScan().useSnapshot(snapshotId).planFiles(); CloseableIterable filteredScanTasks = CloseableIterable.filter(scanTasks, t -> { DataFile file = t.asFileScanTask().file(); - return shouldIncludeForCompaction(table, partitionPath, file, fileSizeThreshold); + return shouldIncludeForCompaction(table, partitionName, file, fileSizeThreshold); }); return Lists.newArrayList(CloseableIterable.transform(filteredScanTasks, t -> t.asFileScanTask().file())); } @@ -84,19 +86,19 @@ public static List getDataFiles(Table table, Long snapshotId, String p /** * Returns table's list of delete files as following: * 1. If the table is unpartitioned, returns all delete files. - * 2. If partitionPath is not provided, returns all delete files that belong to the non-latest partition spec. - * 3. If partitionPath is provided, returns all delete files that belong to corresponding partition. + * 2. If partitionName is not provided, returns all delete files that belong to the non-latest partition spec. + * 3. If partitionName is provided, returns all delete files that belong to corresponding partition. * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name */ - public static List getDeleteFiles(Table table, Long snapshotId, String partitionPath) { + public static List getDeleteFiles(Table table, Long snapshotId, String partitionName) { Table deletesTable = MetadataTableUtils.createMetadataTableInstance(table, MetadataTableType.POSITION_DELETES); CloseableIterable deletesScanTasks = deletesTable.newBatchScan().useSnapshot(snapshotId).planFiles(); CloseableIterable filteredDeletesScanTasks = CloseableIterable.filter(deletesScanTasks, t -> { DeleteFile file = ((PositionDeletesScanTask) t).file(); - return shouldIncludeForCompaction(table, partitionPath, file); + return shouldIncludeForCompaction(table, partitionName, file); }); return Lists.newArrayList(CloseableIterable.transform(filteredDeletesScanTasks, t -> ((PositionDeletesScanTask) t).file())); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java index 387d3d983f76..5e3f4960eec1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java @@ -21,10 +21,8 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.Warehouse; @@ -42,8 +40,10 @@ import org.apache.hadoop.hive.ql.parse.TransformSpec; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext; +import org.apache.hadoop.hive.ql.txn.compactor.CompactorUtil; import org.apache.hadoop.hive.ql.txn.compactor.QueryCompactor; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hive.iceberg.org.apache.orc.storage.common.TableName; import org.apache.iceberg.PartitionField; @@ -71,6 +71,10 @@ public boolean run(CompactorContext context) throws IOException, HiveException, HiveConf conf = new HiveConf(context.getConf()); CompactionInfo ci = context.getCompactionInfo(); + // the settings a table carries for its compactions decide what the query reads and what it + // measures, so they are in hand before either is settled + CompactorUtil.overrideConfProps(conf, ci, tblProperties); + String compactionQuery = buildCompactionQuery(context, compactTableName, conf); SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties); @@ -111,6 +115,15 @@ private String buildCompactionQuery(CompactorContext context, String compactTabl } } + // only a compaction that rewrote the whole table, or a whole partition of a table keeping + // statistics per partition, may store what it measured + boolean computeColStats = ci.isMajorCompaction() && (ci.partName == null ? + !icebergTable.spec().isPartitioned() : + IcebergTableUtil.isPartitionStats(icebergTable, conf)); + + boolean genericStats = HiveConf.getBoolVar(conf, ConfVars.HIVE_COMPACTOR_GATHER_STATS); + HiveConf.setBoolVar(conf, ConfVars.HIVE_STATS_COL_AUTOGATHER, genericStats && computeColStats); + String compactionQuery = (ci.partName == null) ? buildFullTableCompactionQuery(compactTableName, conf, icebergTable, columnsList, fileSizePredicate, orderBy) : @@ -180,7 +193,7 @@ private String buildPartitionCompactionQuery( HiveConf.setBoolVar(conf, ConfVars.HIVE_CONVERT_JOIN, false); conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false); HiveConf.setVar(conf, ConfVars.REWRITE_POLICY, RewritePolicy.PARTITION.name()); - conf.set(IcebergCompactionService.PARTITION_PATH, new Path(ci.partName).toString()); + conf.set(IcebergCompactionService.PARTITION_NAME, ci.partName); PartitionSpec spec; String partitionPredicate; @@ -214,20 +227,22 @@ private String buildPartitionPredicate(CompactionInfo ci, PartitionSpec spec) th Types.StructType partitionType = spec.partitionType(); return partitionType.fields().stream().map(field -> { + String column = HiveUtils.unparseIdentifier(field.name()); String value = partSpecMap.get(field.name()); - String literal = "NULL"; - - if (value != null && !value.equals("null")) { - String type = HiveSchemaUtil.convertToTypeString(field.type()); - PartitionField partitionField = partitionFieldMap.get(field.name()); - TransformSpec transformSpec = TransformSpec.fromString(partitionField.transform().toString(), field.name()); - literal = TypeInfoUtils.convertStringToLiteralForSQL( - HiveIcebergFilterFactory.convertPartitionLiteral(value, transformSpec).toString(), - ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(type)).getPrimitiveCategory()); + + if (value == null || value.equals(ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal)) { + return String.format("`partition`.%s IS NULL", column); } - return String.format("`partition`.%s %s %s", HiveUtils.unparseIdentifier(field.name()), - Objects.equals(literal, "NULL") ? "IS" : "=", literal); + TransformSpec transformSpec = TransformSpec.fromString( + partitionFieldMap.get(field.name()).transform().toString(), field.name()); + TypeInfo type = TypeInfoUtils.getTypeInfoFromTypeString(HiveSchemaUtil.convertToTypeString(field.type())); + + String literal = TypeInfoUtils.convertStringToLiteralForSQL( + HiveIcebergFilterFactory.convertPartitionLiteral(value, transformSpec).toString(), + ((PrimitiveTypeInfo) type).getPrimitiveCategory()); + + return String.format("`partition`.%s = %s", column, literal); }).collect(Collectors.joining(" AND ")); } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java index e3f177a842c8..e3b64c39e191 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java @@ -185,9 +185,9 @@ public static double getDeleteFileRatio(Map parameters) { .orElse(TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT); } - private static Pair getPartitionSpecStructPair(Table table, String partitionPath) + private static Pair getPartitionSpecStructPair(Table table, String partitionName) throws IOException { - if (!table.spec().isPartitioned() || partitionPath == null) { + if (!table.spec().isPartitioned() || partitionName == null) { return null; } PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils @@ -200,10 +200,10 @@ private static Pair getPartitionSpecStructPair(Table table, PartitionSpec spec = table.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); PartitionData partitionData = IcebergTableUtil.toPartitionData(data, Partitioning.partitionType(table), spec.partitionType()); - String path = spec.partitionToPath(partitionData); - return Maps.immutableEntry(path, Pair.of(spec.specId(), data)); + String partName = IcebergTableUtil.toPartitionName(spec, partitionData); + return Maps.immutableEntry(partName, Pair.of(spec.specId(), data)); }) - .filter(e -> e.getKey().equals(partitionPath)) + .filter(e -> e.getKey().equals(partitionName)) .transform(Map.Entry::getValue) .get(0); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java new file mode 100644 index 000000000000..89d439c47070 --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import io.airlift.compress.zstd.ZstdDecompressor; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.IOUtil; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.mr.hive.IcebergTableUtil; +import org.apache.iceberg.puffin.BlobMetadata; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Reads the column statistics {@link IcebergColStatsWriter} stores: table-level entries one blob + * per column, partition entries one framed blob per partition, fetching and decoding only the + * columns asked. + */ +public final class IcebergColStatsReader { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergColStatsReader.class); + + /** Below this, fetching the blob whole costs less than a second positioned read. */ + static final long WHOLE_BLOB_READ_BYTES = 32 * 1024; + private static final long HEADER_PROBE_BYTES = 4 * 1024; + /** Asked slices closer than this are fetched in one read; a seek costs more than the gap. */ + private static final long COALESCE_GAP_BYTES = 64 * 1024; + + private IcebergColStatsReader() { + } + + /** The stored statistics describing the whole table. */ + public static List readColStats(Table table, long snapshotId, Predicate filter) { + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(table, snapshotId, false); + if (statsFile == null) { + LOG.warn("Column stats file not found for snapshot: {}", snapshotId); + return Lists.newArrayList(); + } + try { + return readColStatsOrThrow(table, statsFile, filter); + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to read column stats: {}", e.getMessage()); + return Lists.newArrayList(); + } + } + + /** + * The strict variant for the merge path: an unreadable statistics file must not be mistaken for + * an absent one, or the increment would be persisted as the complete statistics. + */ + static List readColStatsOrThrow(Table table, StatisticsFile statsFile, Predicate filter) + throws IOException { + List colStats = Lists.newArrayList(); + String statsPath = statsFile.path(); + try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)) + .withFileSize(statsFile.fileSizeInBytes()) + .withFooterSize(statsFile.fileFooterSizeInBytes()) + .build()) { + List blobMetadata = reader.fileMetadata().blobs(); + + if (filter != null) { + blobMetadata = blobMetadata.stream().filter(filter) + .toList(); + } + Iterator it = Iterables.transform(reader.readAll(blobMetadata), Pair::second).iterator(); + LOG.info("Using column stats from: {}", statsPath); + + while (it.hasNext()) { + byte[] byteBuffer = ByteBuffers.toByteArray(it.next()); + colStats.add(SerializationUtils.deserialize(byteBuffer)); + } + } + return colStats; + } + + /** + * The stored partition entries the given file holds for the partitions the filter admits, each + * trimmed to the asked columns; a null column set asks for all of them. + */ + public static Map> readPartColStats(Table table, StatisticsFile statsFile, + Predicate partitionFilter, Set columns) { + Map> result = Maps.newLinkedHashMap(); + try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsFile.path())) + .withFileSize(statsFile.fileSizeInBytes()) + .withFooterSize(statsFile.fileFooterSizeInBytes()) + .build()) { + List blobs = reader.fileMetadata().blobs().stream() + .filter(metadata -> IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + metadata.properties().containsKey(IcebergTableUtil.PARTITION_FIELD)) + .filter(metadata -> { + String partName = metadata.properties().get(IcebergTableUtil.PARTITION_FIELD); + return partName != null && (partitionFilter == null || partitionFilter.test(partName)); + }) + .toList(); + LOG.info("Using column stats from: {}", statsFile.path()); + List whole = Lists.newArrayList(); + for (BlobMetadata blob : blobs) { + // a narrow ask on a big blob fetches just its own slices; anything else reads the blob whole + if (columns == null || blob.length() <= WHOLE_BLOB_READ_BYTES) { + whole.add(blob); + } else { + result.put(blob.properties().get(IcebergTableUtil.PARTITION_FIELD), + readSlices(table.io().newInputFile(statsFile.path()), blob.offset(), blob.length(), columns)); + } + } + for (Pair blob : reader.readAll(whole)) { + result.put(blob.first().properties().get(IcebergTableUtil.PARTITION_FIELD), + decodePartitionBlob(blob.second(), columns)); + } + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to read column stats: {}", e.getMessage()); + result.clear(); + } + return result; + } + + /** The slices of the asked columns out of a whole blob, the rest skipped without decoding. */ + static List decodePartitionBlob(ByteBuffer blob, Set columns) + throws IOException { + DataInputStream data = new DataInputStream(new ByteArrayInputStream(ByteBuffers.toByteArray(blob))); + if (data.readInt() != IcebergColStatsWriter.PART_STATS_FORMAT_VERSION) { + // a frame this reader does not know reads as absent rather than wrong + return List.of(); + } + int count = data.readInt(); + data.readInt(); + String[] names = new String[count]; + int[] storedLengths = new int[count]; + int[] rawLengths = new int[count]; + for (int i = 0; i < count; i++) { + names[i] = data.readUTF(); + storedLengths[i] = data.readInt(); + rawLengths[i] = data.readInt(); + } + List statsObjs = Lists.newArrayList(); + for (int i = 0; i < count; i++) { + if (columns == null || columns.contains(names[i])) { + byte[] slice = new byte[storedLengths[i]]; + data.readFully(slice); + statsObjs.add(decodeSlice(slice, rawLengths[i])); + } else { + data.skipNBytes(storedLengths[i]); + } + } + return statsObjs; + } + + /** + * The asked columns of one blob, fetched by position: the header names where each slice sits, so + * only the slices asked for are read at all. Reads coalesce while the gap between asked slices + * costs less than another seek, so projecting 3 of 3000 columns reads about 3 slices. + */ + static List readSlices(InputFile file, long blobOffset, long blobLength, + Set columns) throws IOException { + try (SeekableInputStream in = file.newStream()) { + byte[] probe = new byte[(int) Math.min(blobLength, HEADER_PROBE_BYTES)]; + in.seek(blobOffset); + IOUtil.readFully(in, probe, 0, probe.length); + DataInputStream head = new DataInputStream(new ByteArrayInputStream(probe)); + if (head.readInt() != IcebergColStatsWriter.PART_STATS_FORMAT_VERSION) { + return List.of(); + } + int count = head.readInt(); + int headerLength = head.readInt(); + if (headerLength > probe.length) { + byte[] header = Arrays.copyOf(probe, headerLength); + IOUtil.readFully(in, header, probe.length, headerLength - probe.length); + head = new DataInputStream(new ByteArrayInputStream(header, 3 * Integer.BYTES, + headerLength - 3 * Integer.BYTES)); + } + String[] names = new String[count]; + int[] storedLengths = new int[count]; + int[] rawLengths = new int[count]; + for (int i = 0; i < count; i++) { + names[i] = head.readUTF(); + storedLengths[i] = head.readInt(); + rawLengths[i] = head.readInt(); + } + long[] offsets = new long[count]; + long offset = headerLength; + for (int i = 0; i < count; i++) { + offsets[i] = offset; + offset += storedLengths[i]; + } + List statsObjs = Lists.newArrayList(); + int cursor = 0; + while (cursor < count) { + if (!columns.contains(names[cursor])) { + cursor++; + continue; + } + int first = cursor; + int last = cursor; + for (int next = cursor + 1; next < count; next++) { + if (columns.contains(names[next])) { + if (offsets[next] - (offsets[last] + storedLengths[last]) > COALESCE_GAP_BYTES) { + break; + } + last = next; + } + } + byte[] range = new byte[(int) (offsets[last] + storedLengths[last] - offsets[first])]; + in.seek(blobOffset + offsets[first]); + IOUtil.readFully(in, range, 0, range.length); + for (int slice = first; slice <= last; slice++) { + if (columns.contains(names[slice])) { + statsObjs.add(decodeSlice(Arrays.copyOfRange(range, (int) (offsets[slice] - offsets[first]), + (int) (offsets[slice] - offsets[first]) + storedLengths[slice]), rawLengths[slice])); + } + } + cursor = last + 1; + } + return statsObjs; + } + } + + private static ColumnStatisticsObj decodeSlice(byte[] stored, int rawLength) { + byte[] raw = new byte[rawLength]; + new ZstdDecompressor().decompress(stored, 0, stored.length, raw, 0, rawLength); + return SerializationUtils.deserialize(raw); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java new file mode 100644 index 000000000000..0cf39f6207f7 --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -0,0 +1,496 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import io.airlift.compress.zstd.ZstdCompressor; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BooleanSupplier; +import java.util.function.Predicate; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; +import org.apache.hadoop.hive.ql.Context.RewritePolicy; +import org.apache.hadoop.hive.ql.parse.ColumnStatsSemanticAnalyzer; +import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.mr.hive.IcebergTableUtil; +import org.apache.iceberg.mr.hive.compaction.IcebergCompactionService; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.BlobMetadata; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinCompressionCodec; +import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.puffin.PuffinWriter; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterators; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes the column statistics of one gather as the table's statistics file, per the policy the + * write's facts resolve to: replacing the stored file, merging into it by carrying what no write + * since has changed, or leaving it alone. The reading side is {@link IcebergColStatsReader}. + * + * At table level the file holds one blob per column. At partition level it holds one blob per + * partition, pulled and written one at a time so the whole of a large table's statistics is never + * held at once. A partition blob frames one slice per column behind a small header, so a read + * deserializes only the columns it was asked for: + * + * content := version, count, header length, count x (column name, stored length, raw length), + * slices + * slice := one column's ColumnStatisticsObj, Java-serialized and zstd-compressed on its own + * + * The blob itself is not compressed, and the header names every slice's position, so a reader can + * fetch the columns it was asked for and no others; the table-level blobs, one small one per + * column, stay compressed whole + * + * The frame travels under its own blob type, so a file of the older layout reads as absent rather + * than wrong, and a version bump can change the frame without renaming the type. + */ +public final class IcebergColStatsWriter { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergColStatsWriter.class); + + /** + * The blob type of every entry this writer stores. A table-level entry holds one column bare; a + * partition entry carries the partition name as a property and frames its columns, and the frame + * is versioned apart from the name. + */ + public static final String HIVE_COL_STATS_BLOB_V1 = "hive-column-statistics-v1"; + /** What released writers named a table-level entry; read, never written. */ + public static final String LEGACY_COL_STATS_BLOB = ColumnStatisticsObj.class.getSimpleName(); + static final int PART_STATS_FORMAT_VERSION = 1; + private static final String STATS = "/stats/snap-"; + + private IcebergColStatsWriter() { + } + + /** Everything written describes the snapshot it is written for, so a read asks only what happened after it. */ + public static boolean writeColStats(Table tbl, Snapshot snapshot, Iterator colStats, + Configuration conf) { + List head = Lists.newArrayList(colStats.next()); + WritePolicy policy = WritePolicy.resolve(tbl, snapshot, head, conf); + if (policy == WritePolicy.SKIP) { + return IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf); + } + Iterator all = Iterators.concat(head.iterator(), colStats); + try { + return head.getFirst().getStatsDesc().isIsTblLevel() ? + writeTableColStats(tbl, snapshot, all, policy, conf) : + writePartitionColStats(tbl, snapshot, all, policy, conf); + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to write column stats: {}", e.getMessage()); + return false; + } + } + + private static boolean writeTableColStats(Table tbl, Snapshot snapshot, Iterator colStats, + WritePolicy policy, Configuration conf) throws IOException, InvalidObjectException { + // the table's statistics are one entry holding every column: nothing to stream + ColumnStatistics stats = colStats.next(); + if (policy == WritePolicy.MERGE) { + // A write commits a snapshot of its own, so what it completes sits on the one before it. An + // ANALYZE commits none, but replaces rather than merges, so it never asks. + Long parentId = snapshot.parentId(); + StatisticsFile statsOldSrc = parentId == null ? null : + IcebergTableUtil.getColStatsFile(tbl, parentId, false); + if (statsOldSrc == null) { + // a table-level increment has nothing to add itself to + return false; + } + List statsOld = IcebergColStatsReader.readColStatsOrThrow(tbl, statsOldSrc, null); + if (!statsOld.isEmpty() && !stats.getStatsObj().isEmpty()) { + MetaStoreServerUtils.mergeColStats(stats, new ColumnStatistics(null, statsOld)); + } + } + Schema schema = tbl.spec().schema(); + // a column dropped or renamed since the entry was stored resolves no field: its statistics + // leave with it + stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); + return commitColStatsFile(tbl, snapshot, conf, writer -> { + for (ColumnStatisticsObj obj : stats.getStatsObj()) { + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, + List.of(schema.caseInsensitiveFindField(obj.getColName()).fieldId()), + snapshot.snapshotId(), snapshot.sequenceNumber(), + ByteBuffer.wrap(SerializationUtils.serialize(obj)), + PuffinCompressionCodec.ZSTD, + Map.of())); + } + }); + } + + private static boolean writePartitionColStats(Table tbl, Snapshot snapshot, Iterator colStats, + WritePolicy policy, Configuration conf) throws IOException { + Schema schema = tbl.spec().schema(); + Set written = Sets.newHashSet(); + return commitColStatsFile(tbl, snapshot, conf, writer -> { + boolean first = true; + while (colStats.hasNext()) { + ColumnStatistics stats = colStats.next(); + String partName = stats.getStatsDesc().getPartName(); + if (partName == null) { + // a group naming no partition describes none + continue; + } + // a column dropped or renamed since the entry was stored resolves no field: its + // statistics leave with it + stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); + // only the first blob carries the actual fieldIds, so the footer does not repeat them + // once per partition + List fieldIds = first ? stats.getStatsObj().stream() + .map(obj -> schema.caseInsensitiveFindField(obj.getColName()).fieldId()).toList() : + List.of(-1); + first = false; + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, fieldIds, + snapshot.snapshotId(), snapshot.sequenceNumber(), + encodePartitionBlob(stats.getStatsObj()), + PuffinCompressionCodec.NONE, + Map.of(IcebergTableUtil.PARTITION_FIELD, partName))); + written.add(partName); + } + if (policy == WritePolicy.MERGE) { + carryPartitionColStats(tbl, snapshot, writer, written, conf); + } + }); + } + + /** + * Carries forward, bytes for bytes, the stored entries of the partitions this write never + * measured, as long as no write since the stored file changed them. Carrying is the one place + * that can settle that without a reader paying for the walk, so the walk here is uncapped. + */ + private static void carryPartitionColStats(Table tbl, Snapshot snapshot, PuffinWriter writer, + Set written, Configuration conf) throws IOException { + // an ANALYZE commits no snapshot of its own: it writes to the snapshot it read, where the + // statistics already are, so the walk starts there rather than at the parent + StatisticsFile statsOldSrc = IcebergTableUtil.findColStatsFile(tbl, snapshot.snapshotId(), true); + if (statsOldSrc == null) { + // a partition describes itself: with nothing stored there is nothing to carry, and what + // was computed stands on its own + return; + } + Predicate stillHolds = IcebergTableUtil.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); + try (PuffinReader reader = Puffin.read(tbl.io().newInputFile(statsOldSrc.path())) + .withFileSize(statsOldSrc.fileSizeInBytes()) + .withFooterSize(statsOldSrc.fileFooterSizeInBytes()) + .build()) { + List carried = reader.fileMetadata().blobs().stream() + .filter(metadata -> { + String partName = metadata.properties().get(IcebergTableUtil.PARTITION_FIELD); + return HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + partName != null && !written.contains(partName) && stillHolds.test(partName); + }) + .toList(); + for (Pair blob : reader.readAll(carried)) { + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, List.of(-1), + snapshot.snapshotId(), snapshot.sequenceNumber(), + ByteBuffer.wrap(ByteBuffers.toByteArray(blob.second())), + PuffinCompressionCodec.NONE, + Map.of(IcebergTableUtil.PARTITION_FIELD, + blob.first().properties().get(IcebergTableUtil.PARTITION_FIELD)))); + } + } + } + + @FunctionalInterface + private interface ColStatsBlobWriter { + void write(PuffinWriter writer) throws IOException; + } + + /** Writes one statistics file through the given blobs and commits it for the snapshot. */ + private static boolean commitColStatsFile(Table tbl, Snapshot snapshot, Configuration conf, + ColStatsBlobWriter blobs) throws IOException { + String statsPath = tbl.location() + STATS + UUID.randomUUID(); + StatisticsFile statisticsFile; + try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) + .createdBy(Constants.HIVE_ENGINE) + .build()) { + blobs.write(writer); + writer.finish(); + statisticsFile = new GenericStatisticsFile( + snapshot.snapshotId(), + statsPath, + writer.fileSize(), + writer.footerSize(), + writer.writtenBlobsMetadata().stream() + .map(GenericBlobMetadata::from) + .collect(ImmutableList.toImmutableList())); + } catch (Exception e) { + LOG.warn("Unable to write column stats to the Puffin file: {}", e.getMessage()); + Path path = new Path(statsPath); + FileSystem fs = path.getFileSystem(conf); + if (fs.exists(path)) { + fs.delete(path, false); + } + return false; + } + tbl.updateStatistics() + .setStatistics(statisticsFile) + .commit(); + return true; + } + + /** One zstd-compressed slice per column behind a header naming each slice's position. */ + static ByteBuffer encodePartitionBlob(List statsObjs) throws IOException { + ZstdCompressor compressor = new ZstdCompressor(); + List slices = Lists.newArrayListWithCapacity(statsObjs.size()); + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + DataOutputStream header = new DataOutputStream(headerBytes); + for (ColumnStatisticsObj obj : statsObjs) { + byte[] raw = SerializationUtils.serialize(obj); + byte[] buffer = new byte[compressor.maxCompressedLength(raw.length)]; + int stored = compressor.compress(raw, 0, raw.length, buffer, 0, buffer.length); + header.writeUTF(obj.getColName()); + header.writeInt(stored); + header.writeInt(raw.length); + slices.add(Arrays.copyOf(buffer, stored)); + } + header.flush(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DataOutputStream data = new DataOutputStream(out); + data.writeInt(PART_STATS_FORMAT_VERSION); + data.writeInt(statsObjs.size()); + // the header's own end, so a reader can fetch it without the slices + data.writeInt(3 * Integer.BYTES + headerBytes.size()); + headerBytes.writeTo(data); + for (byte[] slice : slices) { + data.write(slice); + } + data.flush(); + return ByteBuffer.wrap(out.toByteArray()); + } + + /** + * What a write does to the stored column statistics: replace them, merge into them, or leave + * them alone. They live in one file written whole, so every write is one of these three. + */ + enum WritePolicy { + /** Write the computed statistics, discarding the stored ones. */ + REPLACE, + /** + * Write the computed statistics completed by the stored ones: at table level the write's rows + * add to them, at partition level its partitions replace theirs and the rest carry over. + */ + MERGE, + /** Write nothing: leave the stored statistics as they are. */ + SKIP; + + /** + * Everything the decision reads, so that it can be made without a session. {@code statsAccurate} + * is a supplier because answering it walks the table's metadata, and most cases never ask. + */ + record Facts( + // what this write computed + boolean tableWideStats, + // the table it computed them for, and what it already holds for this snapshot + boolean keepsStatsPerPartition, + boolean partitioned, + BooleanSupplier statsAccurate, + // the statement that computed them + boolean analyze, + boolean analyzePartition, + boolean compaction, + boolean majorCompaction, + boolean fullTableRewrite, + boolean singlePartitionRewrite, + // what its commit did to the rows + boolean holdsOnlyAddedRows, + boolean emptySnapshot, + boolean wroteNoRows, + boolean replacePartitions) { + + boolean isStatsAccurate() { + return statsAccurate.getAsBoolean(); + } + } + + /** + * What to do with the statistics a write computed. + * + * @param snapshot the snapshot the statistics describe: the table's current one, or a branch head + * @param colStats the computed statistics, one entry for the table or one per partition + */ + static WritePolicy resolve( + Table tbl, Snapshot snapshot, List colStats, Configuration conf) { + + return resolve(new Facts( + colStats.getFirst().getStatsDesc().isIsTblLevel(), + IcebergTableUtil.isPartitionStats(tbl, conf), + tbl.spec().isPartitioned(), + () -> IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf), + isAnalyze(conf), + isAnalyzePartition(conf), + isCompaction(), + isMajorCompaction(conf), + isFullTableRewrite(conf), + isSinglePartitionRewrite(conf), + holdsOnlyAddedRows(snapshot), + IcebergTableUtil.isEmptySnapshot(snapshot), + wroteNoRows(conf), + Boolean.parseBoolean( + snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP)))); + } + + /** The decision itself, over the facts alone, so that every case of it can be stated as one. */ + static WritePolicy resolve(Facts write) { + if (!write.tableWideStats()) { + return resolveForPartitions(write); + } + if (write.keepsStatsPerPartition()) { + // the table stores its statistics per partition; these describe it as a whole + return SKIP; + } + if (write.compaction()) { + // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. + return write.fullTableRewrite() && write.majorCompaction() && !write.isStatsAccurate() ? + REPLACE : SKIP; + } + // ANALYZE reads the whole table, so it replaces. + return write.analyze() ? REPLACE : resolveForWrite(write); + } + + /** What a statement does to the file holding one blob per partition. */ + private static WritePolicy resolveForPartitions(Facts write) { + if (write.analyze()) { + // what it read is what it named, not what the snapshot it reads happens to hold + return write.analyzePartition() ? MERGE : REPLACE; + } + if (write.compaction()) { + // a partition read whole describes itself; part of one describes none of it + return write.singlePartitionRewrite() && write.majorCompaction() ? MERGE : SKIP; + } + if (write.holdsOnlyAddedRows() || write.emptySnapshot()) { + return REPLACE; + } + if (write.wroteNoRows() || write.isStatsAccurate()) { + return SKIP; + } + // only a write that replaced its partitions measured everything they now hold; an insert + // measured rows it added to partitions holding more, so it may not stand in for them + return write.replacePartitions() ? MERGE : SKIP; + } + + /** What an INSERT, INSERT OVERWRITE or CTAS does to the stored statistics. */ + private static WritePolicy resolveForWrite(Facts write) { + if (write.emptySnapshot()) { + // The table is now empty, so stats of the rows it held must go. Must precede the + // wroteNoRows check, which an emptying write also matches. + return REPLACE; + } + if (write.wroteNoRows() || write.isStatsAccurate()) { + // Nothing to record: the statement wrote no rows, or this snapshot already has stats. + return SKIP; + } + if (write.holdsOnlyAddedRows()) { + // Every row came from this write: a CTAS, a whole-table INSERT OVERWRITE, or the first + // INSERT after a TRUNCATE. + return REPLACE; + } + if (write.replacePartitions()) { + // Iceberg flags whole-table and partition overwrites alike, so the partition spec decides: + // only an unpartitioned table had every row replaced. + return write.partitioned() ? SKIP : REPLACE; + } + // An INSERT: its stats cover the rows it added, the stored ones cover the rest. + return MERGE; + } + + /** + * Whether the table holds nothing but the rows this snapshot added. Iceberg carries the row + * total across commits, so it stays above the added count while older rows remain. False if + * either count is missing from the summary. + */ + private static boolean holdsOnlyAddedRows(Snapshot snapshot) { + String added = snapshot.summary().get(SnapshotSummary.ADDED_RECORDS_PROP); + return added != null && added.equals(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP)); + } + + /** Whether the statement wrote no rows, as its file sink reported to the query state. */ + private static boolean wroteNoRows(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> qs.getNumModifiedRows() == 0) + .orElse(false); + } + + private static boolean isAnalyze(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) + .orElse(false); + } + + /** Whether the ANALYZE named the partitions it is for, leaving the rest of the table alone. */ + private static boolean isAnalyzePartition(Configuration conf) { + return SessionStateUtil.getResource(conf, ColumnStatsSemanticAnalyzer.ANALYZE_PARTITION).isPresent(); + } + + private static boolean isCompaction() { + return SessionState.get() != null && SessionState.get().isCompaction(); + } + + /** Whether the compaction read every file of what it was pointed at: a minor one skips by size. */ + private static boolean isMajorCompaction(Configuration conf) { + return conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + } + + /** Whether what it was pointed at was the whole table, which only an unpartitioned one is. */ + private static boolean isFullTableRewrite(Configuration conf) { + return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)); + } + + /** Whether the compaction was pointed at one partition: a spec-evolution one carries PARTITION too. */ + private static boolean isSinglePartitionRewrite(Configuration conf) { + return conf.get(IcebergCompactionService.PARTITION_NAME) != null; + } + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java index 68a910a950ae..ef073f07a458 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Map; import java.util.stream.LongStream; import org.apache.hadoop.hive.llap.LlapHiveUtils; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; @@ -34,10 +33,10 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.mr.hive.IcebergAcidUtil; -import org.apache.iceberg.util.StructProjection; +import org.apache.iceberg.mr.hive.IcebergTableUtil; /** * Iterator wrapper around Hive's VectorizedRowBatch producer (MRv1 implementing) record readers. @@ -52,17 +51,32 @@ public final class HiveBatchIterator implements CloseableIterator idToConstant; + + private final int specId; + private final long partitionHash; + private final String filePath; + private final String partitionName; + + private final Long firstRowId; + private final Long fileSequenceNumber; HiveBatchIterator(RecordReader recordReader, JobConf job, - int[] partitionColIndices, Object[] partitionValues, Map idToConstant) { + int[] partitionColIndices, Object[] partitionValues, FileScanTask task) { this.recordReader = recordReader; this.key = recordReader.createKey(); this.batch = recordReader.createValue(); this.vrbCtx = LlapHiveUtils.findMapWork(job).getVectorizedRowBatchCtx(); this.partitionColIndices = partitionColIndices; this.partitionValues = partitionValues; - this.idToConstant = idToConstant; + + this.specId = task.file().specId(); + this.partitionHash = IcebergAcidUtil.computeHash(task.file().partition()); + this.filePath = task.file().location(); + this.partitionName = IcebergTableUtil.toPartitionName(task.spec(), task.file().partition()); + + this.firstRowId = task.file().firstRowId(); + this.fileSequenceNumber = task.file().fileSequenceNumber(); + RowLineageUtils.initializeRowLineageColumns(vrbCtx, batch); } @@ -99,24 +113,14 @@ private void advance() { int idx = vrbCtx.findVirtualColumnNum(vc); switch (vc) { case PARTITION_SPEC_ID: - value = idToConstant.get(MetadataColumns.SPEC_ID.fieldId()); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], specId, idx); break; case PARTITION_HASH: - value = IcebergAcidUtil.computeHash( - (StructProjection) idToConstant.get(MetadataColumns.PARTITION_COLUMN_ID)); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], partitionHash, idx); break; case FILE_PATH: - value = idToConstant.get(MetadataColumns.FILE_PATH.fieldId()); BytesColumnVector bcv = (BytesColumnVector) batch.cols[idx]; - if (value == null) { - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; - } else { - bcv.fill(((String) value).getBytes()); - } + bcv.fill(filePath.getBytes()); break; case ROW_POSITION: value = LongStream.range(rowOffset, rowOffset + batch.size).toArray(); @@ -126,16 +130,13 @@ private void advance() { lcv.isRepeating = false; System.arraycopy(value, 0, lcv.vector, 0, batch.size); break; - case PARTITION_PROJECTION: + case PARTITION_NAME: bcv = (BytesColumnVector) batch.cols[idx]; - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; + bcv.fill(partitionName.getBytes()); break; case ROW_LINEAGE_ID: LongColumnVector rowIdLcv = (LongColumnVector) batch.cols[idx]; - Object firstRowIdObj = idToConstant.get(MetadataColumns.ROW_ID.fieldId()); - if (firstRowIdObj == null) { + if (firstRowId == null) { rowIdLcv.noNulls = false; rowIdLcv.isNull[0] = true; rowIdLcv.isRepeating = true; @@ -144,13 +145,13 @@ private void advance() { // If vector[0] is still -1, the reader didn't find the column in the file. if (rowIdLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; } } else { // Lineage data was found (could be 0). Preserve it and fill only the NULL gaps. for (int i = 0; i < batch.size; i++) { if (rowIdLcv.isNull[i]) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; rowIdLcv.isNull[i] = false; } } @@ -161,8 +162,7 @@ private void advance() { case LAST_UPDATED_SEQUENCE_NUMBER: LongColumnVector lusnLcv = (LongColumnVector) batch.cols[idx]; - Object fileSeqObj = idToConstant.get(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); - if (fileSeqObj == null) { + if (fileSequenceNumber == null) { lusnLcv.noNulls = false; lusnLcv.isNull[0] = true; lusnLcv.isRepeating = true; @@ -172,13 +172,13 @@ private void advance() { // If vector[0] is still -1, apply the file-level sequence number to the whole batch. if (lusnLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; } } else { // Lineage data found in file, fill only the gaps where data is missing. for (int i = 0; i < batch.size; i++) { if (!lusnLcv.noNulls && lusnLcv.isNull[i]) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; lusnLcv.isNull[i] = false; } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java index 73ea486a3397..2444132705d0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java @@ -88,7 +88,7 @@ public CloseableIterable filterBatch(CloseableIterable iterator = new DeleteFilterBatchIterator(batches); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java index bfc7cc1fc5ba..0bbc5aa8f08e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java @@ -184,7 +184,7 @@ protected DeleteLoader newDeleteLoader() { } CloseableIterable vrbIterable = - createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, idToConstant); + createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, task); return deleteFilter != null ? deleteFilter.filterBatch(vrbIterable) : vrbIterable; @@ -293,12 +293,12 @@ private static RecordReader parquetRecordReade private static CloseableIterable createVectorizedRowBatchIterable( RecordReader hiveRecordReader, JobConf job, int[] partitionColIndices, - Object[] partitionValues, Map idToConstant) { + Object[] partitionValues, FileScanTask task) { HiveBatchIterator iterator = - new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, idToConstant); + new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, task); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java index 4fe3b533755a..cdb03e1a89c5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java @@ -54,7 +54,7 @@ class HiveIcebergCopyOnWriteRecordWriter extends HiveIcebergDefaultWriter { @Override public void write(Writable row) throws IOException { Record record = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate, false); Record rowData = positionDelete.row(); if (positionDelete.pos() < 0) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java index 9365f9834ab7..6970556fc46a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java @@ -58,7 +58,7 @@ class HiveIcebergDeleteWriter extends HiveIcebergWriterBase { @Override public void write(Writable row) throws IOException { Record rec = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate, isMergeTask); int specId = IcebergAcidUtil.parseSpecId(rec); PartitionKey partitionKey = isMergeTask ? IcebergAcidUtil.parsePartitionKey(rec) : partition(positionDelete.row(), specId); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java index 29f285cf37b1..a0a51b840626 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java @@ -81,7 +81,7 @@ private static Schema readSchema(Configuration conf, Table table, boolean caseSe } if (InputFormatConfig.fetchVirtualColumns(conf)) { - readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns(), table); + readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns()); if (IcebergTableUtil.supportsRowLineage(table.properties())) { readSchema = MetadataColumns.schemaWithRowLineage(readSchema); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java index 8c0dac35f1c1..824acf20e33f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java @@ -26,6 +26,7 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.avro.PlannedDataReader; @@ -57,9 +58,11 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { private CloseableIterator nextTask() { CloseableIterator closeableIterator = openGeneric(mergeSplit.getContentFile(), table.schema()).iterator(); if (mergeSplit.getContentFile() instanceof DeleteFile) { - Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns()); + Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns(), true); + PartitionSpec spec = table.specs().get(mergeSplit.getContentFile().specId()); + return new IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator, - deleteSchema, mergeSplit.getContentFile(), table); + deleteSchema, spec, mergeSplit.getContentFile()); } else { return closeableIterator; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 635a311a6c00..14430fdc3af2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -37,8 +37,6 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; @@ -86,12 +84,13 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { } private CloseableIterator nextTask() { - CloseableIterator closeableIterator = open(tasks.next(), expectedSchema).iterator(); + FileScanTask task = tasks.next(); + CloseableIterator closeableIterator = open(task, expectedSchema).iterator(); if (!isFetchVirtualColumns() || Utilities.getIsVectorized(conf)) { return closeableIterator; } return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator, - expectedSchema, conf); + expectedSchema.columns(), conf, task); } @Override @@ -262,19 +261,12 @@ private CloseableIterable newOrcIterable( } private Map constantsMap(FileScanTask task, BiFunction converter) { - PartitionSpec spec = task.spec(); - Set idColumns = spec.identitySourceIds(); - Schema partitionSchema = TypeUtil.select(expectedSchema, idColumns); - boolean projectsIdentityPartitionColumns = !partitionSchema.columns().isEmpty(); - if (expectedSchema.findField(MetadataColumns.PARTITION_COLUMN_ID) != null) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else if (projectsIdentityPartitionColumns) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else { - return Collections.emptyMap(); + boolean projectsIdentityPartitionColumns = !TypeUtil.select(expectedSchema, task.spec().identitySourceIds()) + .columns().isEmpty(); + if (expectedSchema.findField(MetadataColumns.SPEC_ID.fieldId()) != null || projectsIdentityPartitionColumns) { + return PartitionUtil.constantsMap(task, converter); } + return Collections.emptyMap(); } private static Schema schemaWithoutConstantsAndMeta(Schema readSchema, Map idToConstant) { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java index f82a2f449ade..b1746c21b38e 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; @@ -285,4 +286,20 @@ public void testHistory() throws IOException, InterruptedException { Assert.assertEquals(table.history().get(i).snapshotId(), history.get(i)[0]); } } + + @Test + public void testFilterOnPartitionName() { + // PARTITION__NAME is materialized by the reader and is not a field of the Iceberg schema, so a + // predicate on it must not be pushed down to Iceberg + TableIdentifier identifier = TableIdentifier.of("default", "part_name_filter"); + testTables.createTable(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).identity("last_name").build(), + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); + + List rows = shell.executeStatement( + "SELECT customer_id FROM " + identifier.name() + + " WHERE PARTITION__NAME = 'last_name=Brown' AND customer_id = 0"); + Assert.assertEquals(1, rows.size()); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index df57578b7052..89f1a70ea3aa 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -20,36 +20,55 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DataOperations; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatistics; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.hadoop.ConfigProperties; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsReader; import org.apache.iceberg.mr.hive.test.TestTables; import org.apache.iceberg.mr.hive.test.TestTables.TestTableType; import org.apache.iceberg.mr.hive.test.utils.HiveIcebergStorageHandlerTestUtils; import org.apache.iceberg.mr.hive.test.utils.HiveIcebergTestUtils; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.thrift.TException; import org.junit.Assert; @@ -66,6 +85,9 @@ */ public class TestHiveIcebergStatistics extends HiveIcebergStorageHandlerWithEngineBase { + /** What a partition holding no value for a partition column is named. */ + private static final String NULL_PART = "__HIVE_DEFAULT_PARTITION__"; + @Parameterized.Parameter(4) public String statsSource; @@ -85,6 +107,8 @@ public static Collection parameters() { @Before public void setStatsSource() { HiveConf.setVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE, statsSource); + // these tests describe the per partition statistics an Iceberg table keeps when asked to + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); } @Test @@ -180,6 +204,167 @@ private TableIdentifier getTableIdentifierWithPessimisticLock(String hiveLockEna return identifier; } + @Test + public void testBranchWriteLeavesTableStatsUntouched() { + // a branch write leaves the table's snapshot where it is, so the table's column and basic + // statistics must still describe the table's own rows + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + // answer from the data, not from the statistics under test + shell.setHiveSessionValue("hive.compute.query.using.stats", false); + List mainRows = shell.executeStatement("SELECT max(customer_id) FROM " + identifier); + Assert.assertEquals("the row belongs to the branch, not the table", "2", + String.valueOf(mainRows.get(0)[0])); + + // the table's column statistics still describe the table's rows + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + checkColStat(identifier.name(), "customer_id", true); + + // and the basic statistics likewise count the table's rows, not the branch's + Map basicStats = storageHandler().getBasicStatistics(hmsTable(identifier)); + Assert.assertEquals("3", basicStats.get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(3L), storageHandler().getRowCount(hmsTable(identifier))); + + // and the metastore parameters, which are table-scoped, still count the table's rows + Assert.assertEquals("3", hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + + // asked for the branch, the handler counts the branch's rows + org.apache.hadoop.hive.ql.metadata.Table branchHmsTable = hmsTable(identifier); + branchHmsTable.setSnapshotRef("branch_b1"); + Assert.assertEquals("4", + storageHandler().getBasicStatistics(branchHmsTable).get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(4L), storageHandler().getRowCount(branchHmsTable)); + } + + @Test + public void testAnalyzeOnBranchLeavesTableBasicStatsUntouched() { + // a plain analyze takes the footer scan path, whose row count describes the branch it named + TableIdentifier identifier = TableIdentifier.of("default", "customers_analyze_branch_basic"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + Assert.assertEquals("3", hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS"); + + Assert.assertEquals("the branch's row count is not the table's", "3", + hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(3L), storageHandler().getRowCount(hmsTable(identifier))); + } + + @Test + public void testBranchWriteStoresItsColStatsOnTheBranch() { + // the statistics file is anchored to a snapshot: a branch write stores what it gathered on + // the branch's head, not on the table's + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_stats"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + Table icebergTable = testTables.loadTable(identifier); + long branchSnapshotId = icebergTable.snapshot("b1").snapshotId(); + Assert.assertNotNull("the branch's head carries the statistics its write gathered", + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + + // the increment extends the fork point's statistics: the table's 0..2 plus the branch's 100 + List branchStats = + IcebergColStatsReader.readColStats(icebergTable, branchSnapshotId, null); + ColumnStatisticsObj branchId = branchStats.stream() + .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); + Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(100L, branchId.getStatsData().getLongStats().getHighValue()); + + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + } + + @Test + public void testAnalyzeOnBranchStoresStatsOnTheBranch() { + // an explicit ANALYZE of a branch describes the branch's rows: its statistics belong to the + // branch's head, and the table's own statistics stay as they were + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_analyze_branch"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + // a write that gathers nothing: its head carries no statistics, and 500 is a value neither the + // table nor the statistics the previous write stored have ever seen + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (500, \'Cy\', \'Green\')"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + + long branchSnapshotId = testTables.loadTable(identifier).snapshot("b1").snapshotId(); + Assert.assertNull("the branch's head starts without statistics", IcebergTableUtil.getColStatsFile( + testTables.loadTable(identifier), branchSnapshotId, shell.getHiveConf())); + + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS FOR COLUMNS"); + + // the branch ANALYZE leaves the table's own statistics alone + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + // and it describes the branch's rows, stored on the branch's head + Table icebergTable = testTables.loadTable(identifier); + Assert.assertNotNull("the branch's head carries the statistics the analyze computed", + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + List branchStats = + IcebergColStatsReader.readColStats(icebergTable, branchSnapshotId, null); + ColumnStatisticsObj branchId = branchStats.stream() + .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); + Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals("only the branch holds this row", 500L, + branchId.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testBranchWriteLeavesTableColStatsAccuracyAlone() { + // COLUMN_STATS_ACCURATE describes the table, so a branch write must not restore it + assumeParquetHiveCatalogIceberg(); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_flag"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1),(2)"); + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse("the delete stales the table's statistics", colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (9)"); + Assert.assertFalse("the branch write describes the branch, not the table", + colStatsAccurate(identifier)); + } + @Test public void testStatsWithInsertOverwrite() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -188,6 +373,11 @@ public void testStatsWithInsertOverwrite() { testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + // pre-existing statistics: the overwrite must replace them, not merge onto them + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1, identifier, true); shell.executeStatement(insert); @@ -196,6 +386,324 @@ public void testStatsWithInsertOverwrite() { checkColStatMinMaxValue(identifier.name(), "customer_id", 3, 5); } + @Test + public void testStatsWithPartitionedInsertOverwrite() { + // a partition overwrite replaces the statistics of the partitions it wrote, and carries the + // ones it never reached across unchanged + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_iow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (5, 'a')"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + List colStats = readCurrentColStats(identifier); + // p=a was overwritten, so the rows it no longer holds stop bounding its range + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getHighValue()); + // p=b was never written, so its statistics came across from the previous file + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); + + // and a recompute of the whole table agrees with what the overwrite left behind + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + colStats = readCurrentColStats(identifier); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testPartitionScopedAnalyzeLeavesThePartitionItCannotNameAlone() { + // the rows a table held before it was partitioned belong to a partition no value names, so a + // partition scoped ANALYZE reaches some of them and must not describe that partition by those + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_void"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (100, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(100L, colStatsObj(readCurrentColStats(identifier), DummyPartition.VOID, "id") + .getStatsData().getLongStats().getHighValue()); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals("the partition the statement cannot name keeps describing all of its rows", + 100L, colStatsObj(colStats, DummyPartition.VOID, "id") + .getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("and the one it named describes only the rows that partition holds", + 5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeCanNameThePartitionOfNoValue() { + // the name Hive gives that partition is the one a statement gives back to reach it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_named_null"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, p1 string, p2 string) PARTITIONED BY SPEC (p1, p2) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, 'a', 'x'), (4, 'a', NULL), (7, 'a', 'null')"); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p1='a', p2='" + + NULL_PART + "') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals("only the partition holding no value for p2 was measured", + Set.of("p1=a/p2=" + NULL_PART), colStatsPartitions(testTables.loadTable(identifier))); + // the row holding the text "null" belongs to a partition of its own, which this never named + ColumnStatisticsObj id = colStatsObj(colStats, "p1=a/p2=" + NULL_PART, "id"); + Assert.assertEquals(4L, id.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(4L, id.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresEveryPartitionThePartialSpecNames() { + // naming some of the partition columns names every partition that agrees on them, and the + // statistics of each stand for that partition alone + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_keys"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, p1 string, p2 string) PARTITIONED BY SPEC (p1, p2) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, 'a', 'x'), (9, 'a', 'y'), (7, 'b', 'x'), (4, 'a', NULL)"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p1='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + // the partition holding no value for p2 is named apart from one holding the text of one + Assert.assertEquals("every partition agreeing on p1 was measured, and nothing else", + Set.of("p1=a/p2=x", "p1=a/p2=y", "p1=a/p2=" + NULL_PART), + colStatsPartitions(testTables.loadTable(identifier))); + Assert.assertEquals(1L, colStatsObj(colStats, "p1=a/p2=x", "id") + .getStatsData().getLongStats().getHighValue()); + Assert.assertEquals(9L, colStatsObj(colStats, "p1=a/p2=y", "id") + .getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresATransformPartitionWhole() { + // a value names rows, and the partition holding them holds more; naming it measures all of it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_trunc"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (truncate(1, p)) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'aa'), (9, 'ab'), (7, 'bb')"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='aa') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj truncatedToA = colStatsObj(colStats, "p_trunc=a", "id"); + Assert.assertEquals("the rows the value named are not the only ones the partition holds", + 1L, truncatedToA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(9L, truncatedToA.getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("the partition the statement never named stays undescribed", + Set.of("p_trunc=a"), colStatsPartitions(testTables.loadTable(identifier))); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresTheCurrentSpecPartitionAlone() { + // a write lands in the current spec, so the partitions of an older one keep describing + // themselves, and naming a partition measures the one the table writes today + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_multispec"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p, truncate(1, p))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'a')"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj currentSpec = colStatsObj(colStats, "p=a/p_trunc_1=a", "id"); + Assert.assertEquals(9L, currentSpec.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(9L, currentSpec.getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("the older spec's partitions are not the ones the statement named", + Set.of("p=a/p_trunc_1=a"), colStatsPartitions(testTables.loadTable(identifier))); + } + + @Test + public void testPartitionScopedAnalyzeWithoutStoredColStats() { + // there is nothing stored to carry, so the partition it named is all the file has to hold + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_scoped_first"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + // an insert gathers nothing while the table keeps its statistics per partition + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a"), colStatsPartNames(identifier)); + } + + @Test + public void testPartitionScopedAnalyzeAfterWholeTableOverwrite() { + // the snapshot it reads was written by something that covered the table, which says nothing + // about what the ANALYZE itself was pointed at + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_after_iow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (1, 'a'), (7, 'b')"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + Assert.assertEquals(7L, colStatsObj(readCurrentColStats(identifier), "p=b", "id") + .getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testFullAnalyzeRetiresVanishedPartition() { + // recomputing the whole table is a reset: a partition it no longer finds keeps no statistics, + // where one that named partitions would have carried them over + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_retire"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE p = 'b'"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a"), colStatsPartNames(identifier)); + } + + @Test + public void testStatsWithPartitionScopedAnalyze() { + // an ANALYZE naming a partition recomputes that one and leaves the others as they were + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_scoped"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (9, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(9L, colStatsObj(readCurrentColStats(identifier), "p=a", "id") + .getStatsData().getLongStats().getHighValue()); + + // take the row that bounded p=a away, so only statistics that replace rather than combine can + // report the range that is left + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 9"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(2L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + // p=b was not named, so its statistics came across untouched + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testStatsWithTransformPartition() { + // a transform names its partition by the value it produced, which reads nothing like the sort + // ordinal the transform hands back + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_day"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (day(ts)) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, timestamp'2024-01-01 10:00:00'), (7, timestamp'2024-06-15 12:00:00')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, + colStatsObj(colStats, "ts_day=2024-01-01", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, + colStatsObj(colStats, "ts_day=2024-06-15", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testStatsWithTimestampIdentityPartition() { + // an identity timestamp is one Hive and Iceberg spell differently + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_ts"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, timestamp'2024-01-01 10:00:00'), (7, timestamp'2024-06-15 12:00:00')"); + + checkColStat(identifier.name(), "id", true); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + } + + @Test + public void testStatsWithPartitionFieldsOutOfSchemaOrder() { + // the spec orders its fields as it likes, which the row carrying their values has to follow + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_order"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, b string, c string) " + + "PARTITIONED BY SPEC (c, b) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (1, 'bb', 'cc'), (7, 'bb2', 'cc2')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, colStatsObj(colStats, "c=cc/b=bb", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "c=cc2/b=bb2", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testStatsWithNullPartitionValue() { + // the rows a partition holds none of a value for still form a partition of their own + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_null"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (day(ts)) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, null), (7, timestamp'2024-06-15 12:00:00')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, colStatsObj(colStats, "ts_day=" + NULL_PART, "id") + .getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, + colStatsObj(colStats, "ts_day=2024-06-15", "id").getStatsData().getLongStats().getLowValue()); + } + @Test public void testStatsWithPartitionedInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -213,6 +721,8 @@ public void testStatsWithPartitionedInsert() { String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false); shell.executeStatement(insert); + // partition-level statistics are maintained by complete-scope writers, not inserts + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); checkColStat(identifier.name(), "customer_id", true); checkColStat(identifier.name(), "first_name", true); @@ -252,11 +762,6 @@ public void testStatsWithPartitionedCTAS() { testTables.propertiesForCreateTableSQL( ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())))); - if (statsSource.equals("iceberg")) { - // TODO: Propagate partition spec from CREATE statement to the ColumnStatsSemanticAnalyzer - shell.executeStatement("ANALYZE TABLE target COMPUTE STATISTICS FOR COLUMNS"); - } - checkColStat("target", "id", true); checkColStat("target", "dept", true); checkColStatMinMaxValue("target", "id", 0, 2); @@ -264,6 +769,135 @@ public void testStatsWithPartitionedCTAS() { checkColStatMaxLengthDistinctValue("target", "name", 5, 3); } + @Test + public void testTableLevelColStatsForPartitionedCtas() { + // the create gathered statistics covering every row it wrote: at table granularity those are + // the table's own, so they stand without an analyze to follow + assumeParquetHiveCatalogIceberg(); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE TABLE source (id bigint, name string) PARTITIONED BY (dept string) STORED AS ORC"); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, TableIdentifier.of("default", "source"), false)); + + shell.executeStatement(String.format( + "CREATE TABLE target PARTITIONED BY (dept, name) STORED BY ICEBERG %s AS SELECT * FROM source s", + testTables.propertiesForCreateTableSQL( + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())))); + + checkColStat("target", "id", true); + checkColStatMinMaxValue("target", "id", 0, 2); + } + + @Test + public void testTableLevelColStatsForPartitionedInsertOverwrite() { + // an overwrite of the whole table recomputes every row, so at table granularity its statistics + // are the table's; one scoped to a partition leaves the rows of the others behind + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "iow_source"); + TableIdentifier identifier = TableIdentifier.of("default", "iow_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint, p string) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + source + " VALUES (11, \'a\'), (19, \'b\')"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) PARTITIONED BY (p string) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'a\') VALUES (1), (2)"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'b\') VALUES (7), (9)"); + + shell.executeStatement( + "INSERT OVERWRITE TABLE " + identifier + " PARTITION (p) SELECT id, p FROM " + source); + Assert.assertTrue("the overwrite rewrote every row", colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 11, 19); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " PARTITION (p=\'a\') VALUES (100)"); + Assert.assertFalse("a partition overwrite describes a slice, not the table", + colStatsAccurate(identifier)); + } + + @Test + public void testColStatsForInsertOverwriteEmptyingTheTable() { + // an overwrite that selects nothing empties an unpartitioned table: the statistics describing + // the rows it held must not outlive them + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "iow_empty_source"); + TableIdentifier identifier = TableIdentifier.of("default", "iow_empty_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (2), (3)"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 3); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " SELECT id FROM " + source); + + Assert.assertEquals("the overwrite left no rows", Long.valueOf(0L), + storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the empty table's statistics are its own", colStatsAccurate(identifier)); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + } + + @Test + public void testColStatsForCtasSelectingNoRows() { + // the create wrote no rows: statistics describing none are the ones the table has + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "ctas_empty_source"); + TableIdentifier identifier = TableIdentifier.of("default", "ctas_empty_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + source + " VALUES (1), (2)"); + shell.executeStatement("CREATE TABLE " + identifier.name() + + " STORED BY ICEBERG STORED AS PARQUET AS SELECT id FROM " + source + " WHERE id < 0"); + + // writing no rows commits no snapshot, so the table has no state for statistics to describe + Assert.assertNull(testTables.loadTable(identifier).currentSnapshot()); + Assert.assertNull(storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertFalse("no statistics stand for a table that holds nothing", + colStatsAccurate(identifier)); + } + + @Test + public void testColStatsAfterEmptyingPartitionedTable() { + // a delete of every row and a truncate both leave the table empty: the statistics of the rows + // they removed must not outlive them + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_emptied"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) PARTITIONED BY (p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2', 'external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'a\') VALUES (1), (2)"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'b\') VALUES (7), (9)"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 9); + + shell.executeStatement("DELETE FROM " + identifier); + Assert.assertEquals(Long.valueOf(0L), storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the walk stops at the emptied snapshot", readCurrentColStats(identifier).isEmpty()); + Assert.assertFalse(colStatsAccurate(identifier)); + + // and the next write re-anchors the chain rather than extending what the delete left + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'c\') VALUES (20)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 20, 20); + + shell.executeStatement("TRUNCATE TABLE " + identifier); + Assert.assertEquals(Long.valueOf(0L), storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the truncate leaves nothing to serve", readCurrentColStats(identifier).isEmpty()); + } + @Test public void testStatsRemoved() throws IOException { Assume.assumeTrue("Only HiveCatalog can remove stats which become obsolete", @@ -314,6 +948,31 @@ public void testColumnStatsAccurate() throws Exception { } } + @Test + public void testTableLevelColStatsTakeInWhatEachInsertAdds() { + // statistics kept for the table as a whole take in what a write adds to it, so a table + // described in full stays described as rows arrive + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_incr"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (100, 'c')"); + + Assert.assertTrue("the table stays described, having taken the write in", + colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 100); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, true); + } + @Test public void testMergeStatsWithInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -360,7 +1019,8 @@ public void testIcebergColStatsPath() throws IOException { table.refresh(); - Path tblColPath = new Path(IcebergTableUtil.getColStatsPath(table)); + Path tblColPath = new Path(IcebergTableUtil.getColStatsFile( + table, table.currentSnapshot().snapshotId(), shell.getHiveConf()).path()); Assert.assertNotNull(tblColPath); // Check that if colPath is created correctly Assert.assertTrue(tblColPath.getFileSystem(shell.getHiveConf()).exists(tblColPath)); @@ -370,7 +1030,7 @@ public void testIcebergColStatsPath() throws IOException { } @Test - public void testGetAggrBasicStatsForPartitioned() { + public void testGetAggrBasicStatsForPartitioned() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -404,27 +1064,47 @@ public void testAnalyzePartitionSpecRejected() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); createPartitionedCustomers(identifier); - String expected = ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(); - // basic statistics are maintained incrementally for all partitions as a whole: - // a partition-scoped basic-stats ANALYZE cannot be honored and must be rejected + // the metastore holds one row count for the table, with nowhere to record a single partition's AssertHelpers.assertThrows( "Should reject partition-scoped basic-stats ANALYZE for non-native partitioned tables", - IllegalArgumentException.class, expected, + IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS") ); - // same for column statistics: the rewrite would drop every other partition's column stats + // and column statistics of one partition need a table that keeps them per partition + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); AssertHelpers.assertThrows( - "Should reject partition-scoped column-stats ANALYZE for non-native partitioned tables", - IllegalArgumentException.class, expected, + "Should reject partition-scoped column-stats ANALYZE when statistics are kept for the table", + IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS FOR COLUMNS") ); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, true); + } + + @Test + public void testStatsWithPartitionScopedInsertOverwrite() { + // naming the partition to overwrite reaches the same statistics as letting the rows choose it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_static"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " PARTITION (p='a') VALUES (5)"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getHighValue()); } @Test - public void testAnalyzeCatchesUpPartitionStats() { + public void testAnalyzeCatchesUpPartitionStats() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -462,6 +1142,7 @@ public void testGetAggrColStatsForPartitioned() throws Exception { TableIdentifier identifier = TableIdentifier.of("default", "customers"); createPartitionedCustomers(identifier); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); HiveIcebergStorageHandler handler = storageHandler(); @@ -477,6 +1158,44 @@ public void testGetAggrColStatsForPartitioned() throws Exception { // customer ids 0..2, one per last_name partition, merged across the three partitions Assert.assertEquals(0, statsObj.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(2, statsObj.getStatsData().getLongStats().getHighValue()); + + // single-partition probes pin each blob's content to its name, which the span check cannot + AggrStats brown = handler.getAggrColStatsFor(hmsTable, ImmutableList.of("customer_id"), + List.of("last_name=Brown")); + Assert.assertEquals(1, brown.getPartsFound()); + Assert.assertEquals(0, brown.getColStats().get(0).getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(0, brown.getColStats().get(0).getStatsData().getLongStats().getHighValue()); + AggrStats pink = handler.getAggrColStatsFor(hmsTable, ImmutableList.of("customer_id"), + List.of("last_name=Pink")); + Assert.assertEquals(1, pink.getPartsFound()); + Assert.assertEquals(2, pink.getColStats().get(0).getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2, pink.getColStats().get(0).getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testGetAggrColStatsForNullAndEmptyPartitions() throws Exception { + // NULL and empty-string partition values render apart, and do so on both the + // blob-write side (ANALYZE) and the pruned-name side; a rendering mismatch silently drops the + // partition from the aggregation and partial aggregation extrapolates fabricated NDVs + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createPartitionedCustomers(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'Alice', NULL), (4, 'Eve', '')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + + List partNames = partitionNames(handler, hmsTable); + Assert.assertEquals(partNames.toString(), 5, partNames.size()); + Assert.assertTrue(partNames.toString(), partNames.contains("last_name=" + NULL_PART)); + Assert.assertTrue(partNames.toString(), partNames.contains("last_name=")); + + // the blobs must carry the read side's names: an empty-string value that decodes as null would + // pass the aggregate below by double-serving the null partition's key + Assert.assertEquals(partNames.stream().sorted().toList(), colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "customer_id", partNames, 0, 4); } @Test @@ -506,11 +1225,36 @@ public void testCountStarPartitioned() { } @Test - public void testCountStarWithoutPartitionStatsFile() { + public void testBranchWriteComputesItsOwnPartitionStats() { + // the partition statistics file is anchored to a snapshot: a write to a branch computes it + // against the branch's head, leaving the table's own file and head where they were assumeParquetHiveCatalogIceberg(); - TableIdentifier identifier = TableIdentifier.of("default", "customers"); - createPartitionedCustomers(identifier, false); + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_part"); + createPartitionedCustomers(identifier); + + Table icebergTable = testTables.loadTable(identifier); + long tableSnapshotId = icebergTable.currentSnapshot().snapshotId(); + Assert.assertNotNull(IcebergTableUtil.getPartitionStatsFile(icebergTable, tableSnapshotId)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + icebergTable.refresh(); + Assert.assertEquals("the branch write leaves the table's head where it was", + tableSnapshotId, icebergTable.currentSnapshot().snapshotId()); + Assert.assertNotNull("the table keeps the partition statistics describing it", + IcebergTableUtil.getPartitionStatsFile(icebergTable, tableSnapshotId)); + Assert.assertNotNull("the branch's head carries the statistics its write computed", + IcebergTableUtil.getPartitionStatsFile(icebergTable, icebergTable.snapshot("b1").snapshotId())); + } + + @Test + public void testCountStarWithoutPartitionStatsFile() { + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createPartitionedCustomers(identifier, false); Table icebergTable = testTables.loadTable(identifier); Assert.assertNull(IcebergTableUtil.getPartitionStatsFile( @@ -526,7 +1270,7 @@ public void testCountStarWithoutPartitionStatsFile() { } @Test - public void testRowCountWithDeletes() { + public void testRowCountWithDeletes() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -572,21 +1316,24 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { IcebergTableUtil.readPartitionStats(icebergTable, icebergTable.currentSnapshot()); Assert.assertEquals(3L, fileStats.get(DummyPartition.VOID).dataRecordCount().longValue()); - // column stats blobs are written for the physical partitions only: values existing solely among the - // legacy unpartitioned rows (Green, Pink) get no blob + // column stats blobs are written per physical partition; the legacy unpartitioned rows share one + // blob under the synthetic partition name, so values existing solely among them (Green, Pink) + // are accounted there List colStats = - IcebergTableUtil.readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId(), null); + readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); Assert.assertEquals( - List.of("last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), + List.of(DummyPartition.VOID, + "last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList()); + // the legacy blob covers exactly the rows written before the table was partitioned + ColumnStatisticsObj legacyCustomerId = colStatsObj(colStats, DummyPartition.VOID, "customer_id"); + Assert.assertEquals(0L, legacyCustomerId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, legacyCustomerId.getStatsData().getLongStats().getHighValue()); + // a blob describes the physical partition's files only: the legacy Brown row (customer_id 0) does not // merge into the Brown partition's blob, which covers just the new-spec row (customer_id 3) - ColumnStatisticsObj brownCustomerId = colStats.stream() - .filter(stats -> "last_name=Brown".equals(stats.getStatsDesc().getPartName())) - .flatMap(stats -> stats.getStatsObj().stream()) - .filter(obj -> "customer_id".equals(obj.getColName())) - .findFirst().orElseThrow(); + ColumnStatisticsObj brownCustomerId = colStatsObj(colStats, "last_name=Brown", "customer_id"); Assert.assertEquals(3L, brownCustomerId.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(3L, brownCustomerId.getStatsData().getLongStats().getHighValue()); @@ -620,7 +1367,31 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { } @Test - public void testRowCountAfterEvolutionFromUnpartitioned() { + public void testAggrColStatsAfterEvolutionFromUnpartitioned() throws Exception { + // the legacy unpartitioned-spec rows are computed by a dedicated ANALYZE arm and stored under the + // synthetic partition's blob, so an aggregation over a pruned list holding the synthetic partition + // is complete: no extrapolation, and the legacy rows' values are accounted for + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createEvolvedCustomers(identifier); + // a genuine NULL partition value: its group's partition tuple is all null, exactly like the + // legacy rows' - only the spec id may tell them apart + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'Nia', NULL)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + List statNames = Lists.newArrayList(partitionNames(handler, hmsTable)); + statNames.add(DummyPartition.VOID); + Assert.assertTrue(statNames.toString(), statNames.contains("last_name=" + NULL_PART)); + + // customer ids 0..2 exist only among the legacy unpartitioned rows, 3..6 in the partitioned ones + assertAggrColStatsRange(identifier, "customer_id", statNames, 0, 6); + } + + @Test + public void testRowCountAfterEvolutionFromUnpartitioned() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -655,6 +1426,906 @@ public void testRowCountAfterEvolutionFromUnpartitioned() { Assert.assertEquals(3L, brown.get(1)[0]); } + @Test + public void testAnalyzeColStatsInBatches() { + // guards the single-batch persist exemption: the storage handler holds one statistics file per + // snapshot, so honoring hive.stats.max.num.stats would let every batch replace the previous one + // (this cap would force one partition per batch and only the last would survive) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createPartitionedCustomers(identifier); + // 3 stats objects per partition (customer_id, first_name, last_name): one partition per batch + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_MAX_NUM_STATS.varname, "3"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertEquals( + List.of("last_name=Brown", "last_name=Green", "last_name=Pink"), + colStatsPartNames(identifier)); + } + + @Test + public void testAggrColStatsAfterPartitionedSpecEvolution() { + // two partitioned specs: every row is grouped and its blob named under the spec that wrote it, + // in a single ANALYZE pass (the per-spec union rewrite could not even compile - the branches' + // partition structs had different field names) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) + .identity("last_name").build(); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, + fileFormat, ImmutableList.of(), formatVersion); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (first_name)"); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1, identifier, false)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertEquals( + List.of("first_name=Laci", "first_name=Marci", "first_name=Peti", + "last_name=Brown", "last_name=Green", "last_name=Pink"), + colStatsPartNames(identifier)); + } + + @Test + public void testAggrColStatsForYearTransformPartitions() throws Exception { + // time-transform partition values must render as Iceberg's human form ("2023"), not the raw + // transform ordinal ("53"): statistics and partition pruning join on the rendered name + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_by_year"); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), " + + "(2, date '2023-11-11'), (3, date '2024-06-01'), (4, date '1969-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // partition names as the read side renders them via partitionToPath (getPartitions cannot list + // non-identity transforms - its partition filter only supports identity columns); the year + // ordinal is negative for pre-1970 dates + List partNames = ImmutableList.of("d_year=1969", "d_year=2023", "d_year=2024"); + assertAggrColStatsRange(identifier, "id", partNames, 1, 4); + } + + @Test + public void testAggrColStatsForTimestampIdentityPartitions() throws Exception { + // identity-partitioned timestamps: Hive renders the value with a space separator, the blob name + // must carry Iceberg's ISO rendering (Conversions.fromPartitionString cannot parse timestamps) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_by_ts"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, timestamp '2024-06-01 10:00:00'), (2, timestamp '2024-06-01 10:00:00'), " + + "(3, timestamp '2023-11-11 23:59:59')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // partitionToPath URL-escapes the ISO rendering (':' -> %3A) + List partNames = ImmutableList.of("ts=2023-11-11T23%3A59%3A59", "ts=2024-06-01T10%3A00%3A00"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAggrColStatsForCaseSensitivePartitionField() throws Exception { + // Hive's makePartName lowercases the wire keys, so the decode must match the partition field + // case-insensitively: a case-preserving field name (e.g. Spark-created) would otherwise decode + // every group's value to null + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_case"); + Schema schema = new Schema( + NestedField.optional(1, "id", Types.LongType.get()), + NestedField.optional(2, "eventDate", Types.DateType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("eventDate").build(); + testTables.createTable(shell, identifier.name(), schema, spec, fileFormat, ImmutableList.of(), formatVersion); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (2, date '2024-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = ImmutableList.of("eventDate=2023-03-04", "eventDate=2024-06-01"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 2); + } + + @Test + public void testAggrColStatsForTimestampLocalTZIdentityPartitions() throws Exception { + // identity-partitioned zoned timestamps: Hive renders the group value with a trailing zone id + // ("2024-06-01 10:00:00.0 UTC"), which the decode must map back to the instant's micros + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_by_ltz"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, ts timestamp with local time zone) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, timestamp '2024-06-01 10:00:00'), (2, timestamp '2024-06-01 10:00:00'), " + + "(3, timestamp '2023-11-11 23:59:59')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + List partNames = partitionNames(storageHandler(), hmsTable); + Assert.assertEquals(2, partNames.size()); + Assert.assertEquals(partNames.stream().sorted().toList(), colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAggrColStatsForTimeTransformEvolutions() throws Exception { + // year -> month -> day evolutions: one ANALYZE pass names each group's blob with the human + // rendering of the owning spec's transform ordinal + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_by_time"); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (month(d))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (day(d))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, date '2024-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = List.of("d_day=2024-06-01", "d_month=2023-11", "d_year=2023"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAutoGatherSkipsPartitionedInsert() { + // partition-level statistics are maintained by complete-scope writers only: with autogather + // on, a plain INSERT into a partitioned table persists nothing and the analyzed file carries + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_autogather"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11'), " + + "(3, date '2024-06-01')"); + // no new statistics file: the analyzed one keeps serving as an approximation + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023"), colStatsPartNames(identifier)); + } + + @Test + public void testTableLevelColStatsFallbackForPartitioned() throws Exception { + // hive.iceberg.stats.collect.partlevel=false trades partition granularity for cheap + // maintenance: ANALYZE and autogather keep a single table-level file for the partitioned + // table, inserts merge into it incrementally, and planning serves it over the pruned set + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_tbl_level"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (3, date '2023-03-04')"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + // the file is table-level shaped: no blob carries a partition name + Assert.assertTrue(testTables.loadTable(identifier).statisticsFiles().stream() + .flatMap(statsFile -> statsFile.blobMetadata().stream()) + .noneMatch(blob -> blob.properties().containsKey(IcebergTableUtil.PARTITION_FIELD))); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, date '2024-05-05')"); + // the increment merged into the table-level statistics + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + + // the partition-level aggregation finds nothing to serve, at no read cost: the planner + // falls back to the table-level statistics + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable, ImmutableList.of("id"), + partitionNames(storageHandler(), hmsTable)); + Assert.assertEquals(0, aggrStats.getPartsFound()); + Assert.assertTrue(aggrStats.getColStats().isEmpty()); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse(colStatsAccurate(identifier)); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 3, 5); + } + + @Test + public void testAnalyzeReanchorsAfterStatsGap() { + // snapshots committed without statistics leave the stored file behind: the previous + // statistics keep serving as approximations, and ANALYZE re-anchors accounting every row + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_gap"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, date '2023-06-01')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + // the pre-gap statistics keep serving + Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List colStats = readCurrentColStats(identifier); + // after ANALYZE the statistics account for every row + ColumnStatisticsObj id2023 = colStatsObj(colStats, "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(3L, id2023.getStatsData().getLongStats().getHighValue()); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testColStatsServedButFrozenAfterDelete() { + // DML clears the accuracy flag: increments stop extending the statistics (ACID rule), the + // pre-delete file keeps serving as an approximation, and ANALYZE recomputes exactly + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_after_delete"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (2, date '2023-04-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + // the pre-delete statistics keep serving as an approximation + ColumnStatisticsObj id2023 = colStatsObj(readCurrentColStats(identifier), "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getHighValue()); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, date '2024-05-05')"); + // an insert maintains no partition-level statistics: the pre-delete file keeps serving + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023"), colStatsPartNames(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023", "d_year=2024"), colStatsPartNames(identifier)); + // recomputed: the deleted row no longer bounds the range + id2023 = colStatsObj(readCurrentColStats(identifier), "d_year=2023", "id"); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getHighValue()); + } + + + @Test + public void testColStatsNotAccurateAfterExternalWrite() { + // an engine that maintains no Hive statistics can commit at any time: the accuracy flag is + // trusted only while the current snapshot carries its own statistics file + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_external"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + // a foreign commit: new snapshot, no statistics file, no flag maintenance + testTables.loadTable(identifier).newAppend().commit(); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-06-01')"); + // inserts maintain no partition-level statistics; the pre-existing ones keep serving as + // approximations until ANALYZE recomputes + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testSubsetColumnAnalyzeReplacesFile() { + // ANALYZE FOR COLUMNS on a subset replaces the statistics file whole: only the analyzed + // columns' statistics remain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_subset"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (4, date '2023-04-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS id"); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj id2023 = colStatsObj(colStats, "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(4L, id2023.getStatsData().getLongStats().getHighValue()); + Assert.assertTrue(colStats.stream() + .flatMap(stats -> stats.getStatsObj().stream()) + .noneMatch(obj -> "d".equals(obj.getColName()))); + } + + @Test + public void testAnalyzeRecomputesAfterDml() { + // DML stales the statistics; a table-wide ANALYZE recomputes every partition exactly and + // restores the accuracy flag + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_part_analyze"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 2"); + Assert.assertFalse(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List colStats = readCurrentColStats(identifier); + // p=a recomputed exactly: the deleted row no longer bounds it + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(1L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(1L, idA.getStatsData().getLongStats().getHighValue()); + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testColStatsSurviveDataNeutralRewrite() { + // a rewrite that changes no rows (compaction, Hive's or a foreign engine's) commits a + // "replace" snapshot: it neither outdates served statistics nor breaks the merge chain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_rewritten"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1)"); + Assert.assertTrue(colStatsAccurate(identifier)); + + commitDataNeutralRewrite(identifier); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2)"); + // the increment merged across the replace snapshot instead of being dropped + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 2); + } + + /** Rewrites the table's single data file into a byte-identical copy: a "replace" commit. */ + private void commitDataNeutralRewrite(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + try (CloseableIterable tasks = icebergTable.newScan().planFiles()) { + DataFile dataFile = tasks.iterator().next().file(); + String copyPath = dataFile.location() + "-copy"; + try (InputStream in = icebergTable.io().newInputFile(dataFile.location()).newStream(); + OutputStream out = icebergTable.io().newOutputFile(copyPath).create()) { + in.transferTo(out); + } + DataFile copy = DataFiles.builder(icebergTable.spec()) + .copy(dataFile) + .withPath(copyPath) + .build(); + icebergTable.newRewrite() + .rewriteFiles(Set.of(dataFile), Set.of(copy)) + .commit(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + Assert.assertEquals(DataOperations.REPLACE, + testTables.loadTable(identifier).currentSnapshot().operation()); + } + + @Test + public void testEmptyWriteKeepsColStats() { + // an insert that adds no files commits no snapshot: the statistics it computed describe + // nothing and must not replace the stored ones + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_empty_write"); + TableIdentifier source = TableIdentifier.of("default", "orders_empty_src"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("CREATE EXTERNAL TABLE " + source + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + long snapshotId = testTables.loadTable(identifier).currentSnapshot().snapshotId(); + + shell.executeStatement("INSERT INTO " + identifier + " SELECT id FROM " + source); + + // no commit, so the statistics of the unchanged snapshot keep serving, accurate + Assert.assertEquals(snapshotId, testTables.loadTable(identifier).currentSnapshot().snapshotId()); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { + // the same insert onto a table that carries no statistics: an increment gathered over no rows + // describes nothing, so it must not become the table's statistics either + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_empty_write_unanalyzed"); + TableIdentifier source = TableIdentifier.of("default", "orders_empty_write_src"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("CREATE EXTERNAL TABLE " + source + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("INSERT INTO " + identifier + " SELECT id FROM " + source); + + Assert.assertTrue("a write over no rows must not publish statistics", + readCurrentColStats(identifier).isEmpty()); + Assert.assertFalse(colStatsAccurate(identifier)); + } + + @Test + public void testCarriedPartitionColStatsAreAnchoredByWhetherTheyStillHold() { + // ANALYZE full table -> DML on two partitions -> ANALYZE one of them. The partition the DML + // never touched is carried into the new file; the one it reached is not, having stopped + // describing itself. + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_anchor"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + + // one write reaching p=a and p=b, leaving p=c alone + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'a'), (9, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + icebergTable.refresh(); + Assert.assertEquals("the write reached p=b, so what was stored for it is not carried", + ImmutableSet.of("p=a", "p=c"), colStatsPartitions(icebergTable)); + Assert.assertEquals("everything carried is described by the file being written", + icebergTable.currentSnapshot().snapshotId(), currentColStatsFile(icebergTable).snapshotId()); + Assert.assertEquals("and each of them still describes its partition", + ImmutableMap.of("p=a", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testEvolvedFromUnpartitionedDescribesTheSyntheticPartition() throws Exception { + // rows written before the table was partitioned belong to a partition of their own, and both + // the list a scan prunes to and the statistics stored have to name it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_void"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (100, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + + // the pruning path asks for every spec, which is how a scan reaches the legacy rows + List pruned = storageHandler().getPartitions(hmsTable).stream() + .map(org.apache.hadoop.hive.ql.metadata.Partition::getName) + .toList(); + Assert.assertTrue("the pruned list has to name the legacy rows' partition: " + pruned, + pruned.contains(DummyPartition.VOID)); + Assert.assertTrue("and the statistics have to describe it: " + colStatsPartitions(icebergTable), + colStatsPartitions(icebergTable).contains(DummyPartition.VOID)); + + // a write to one partition and an ANALYZE naming only that one: the rest are carried + shell.executeStatement("INSERT INTO " + identifier + " VALUES (7, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + icebergTable.refresh(); + + Assert.assertTrue("the legacy rows' partition has to survive an ANALYZE that never named it: " + + colStatsPartitions(icebergTable), + colStatsPartitions(icebergTable).contains(DummyPartition.VOID)); + } + + @Test + public void testPartitionColStatsSurviveAWriteToAnotherPartition() { + // a write reaches one partition, and the ones it never touched still describe themselves + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_untouched"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals(ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), + upToDateByPartition(icebergTable)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE p = 'a'"); + icebergTable.refresh(); + Assert.assertEquals("only the partition the delete reached stops describing itself", + ImmutableMap.of("p=a", false, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testPartitionColStatsGoStaleOnEveryFormatVersion() { + // a version 1 table numbers every snapshot 0, so sequence numbers cannot order them; where + // they sit on the ancestry can, and that is what every version is read by + assumeParquetHiveCatalogIceberg(); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + + for (int version = 1; version <= 3; version++) { + TableIdentifier identifier = TableIdentifier.of("default", "orders_format_v" + version); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='" + version + "')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + if (version == 1) { + Assert.assertEquals("every snapshot of a version 1 table shares one sequence number", + 0, icebergTable.currentSnapshot().sequenceNumber()); + } + Assert.assertEquals("version " + version + ": what was just computed describes the table", + ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'b')"); + icebergTable.refresh(); + Assert.assertEquals("version " + version + ": only the partition the insert reached stops " + + "describing itself", + ImmutableMap.of("p=a", true, "p=b", false, "p=c", true), upToDateByPartition(icebergTable)); + } + } + + @Test + public void testAggrColStatsCountsOnlyPartitionsCarryingEveryColumnAsked() throws Exception { + // a partition described for only some of the columns asked about does not count as found + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_added_column"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = ImmutableList.of("p=a", "p=b"); + Assert.assertEquals("both partitions carry the column asked about", 2, + storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("id"), partNames) + .getPartsFound()); + + // a column added after the statistics were stored is described by no entry of theirs + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (amount bigint)"); + AggrStats aggrStats = storageHandler().getAggrColStatsFor( + hmsTable(identifier), ImmutableList.of("id", "amount"), partNames); + Assert.assertEquals("a partition missing one of the columns asked about is not counted", + 0, aggrStats.getPartsFound()); + } + + @Test + public void testTableLevelColStatsAreWithheldWhenAWriteLeftThemBehind() { + // judge the granularity served: a current per partition file says nothing about whole-table + // numbers an older snapshot holds + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_granularities"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertFalse("whole-table numbers were stored and nothing has happened since", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertTrue("the insert left the whole-table numbers behind, per partition ones aside", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + } + + @Test + public void testRowPreservingCommitsDoNotSpendTheSnapshotLookback() { + // the lookback bounds manifest reads, and a commit that moves no rows costs none of it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_rewritten"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setIntVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK, 1); + Table icebergTable = testTables.loadTable(identifier); + for (int rewrite = 0; rewrite < 3; rewrite++) { + icebergTable.rewriteManifests().clusterBy(file -> "all").commit(); + } + icebergTable.refresh(); + Assert.assertEquals(DataOperations.REPLACE, icebergTable.currentSnapshot().operation()); + Assert.assertEquals("more rewrites than the lookback allows still leave the statistics placed", + ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testUpdateStalesThePartitionItLeftAndTheOneItReached() { + // moving a row between partitions deletes from one and writes to the other, and afterwards + // neither of them describes itself + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_moved"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals(ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), + upToDateByPartition(icebergTable)); + + shell.executeStatement("UPDATE " + identifier + " SET p = 'b' WHERE id = 1"); + icebergTable.refresh(); + Assert.assertEquals("the partition it took the row from and the one it put it in both changed", + ImmutableMap.of("p=a", false, "p=b", false, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testCopyOnWriteDeleteStalesOnlyTheRewrittenPartition() { + // a copy-on-write delete rewrites the files of the partition it touches rather than adding + // delete files, so the walk has to read what the snapshot replaced, not only what it removed + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_cow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2', 'write.delete.mode'='copy-on-write')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals("only the partition whose files were rewritten stops describing itself", + ImmutableMap.of("p=a", false, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + /** The statistics file a read of the table serves. */ + private StatisticsFile currentColStatsFile(Table icebergTable) { + return IcebergTableUtil.findColStatsFile( + icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf()); + } + + /** The partitions the current statistics file describes, whether or not they still hold. */ + private Set colStatsPartitions(Table icebergTable) { + return currentColStatsFile(icebergTable).blobMetadata().stream() + .map(metadata -> metadata.properties().get(IcebergTableUtil.PARTITION_FIELD)) + .collect(Collectors.toSet()); + } + + /** Whether each stored partition's statistics still describe it at the current snapshot. */ + private Map upToDateByPartition(Table icebergTable) { + StatisticsFile statsFile = currentColStatsFile(icebergTable); + Predicate upToDate = IcebergTableUtil.upToDateColStats( + icebergTable, icebergTable.currentSnapshot(), statsFile, shell.getHiveConf(), true); + return statsFile.blobMetadata().stream().collect(Collectors.toMap( + metadata -> metadata.properties().get(IcebergTableUtil.PARTITION_FIELD), + metadata -> upToDate.test(metadata.properties().get(IcebergTableUtil.PARTITION_FIELD)), + (first, second) -> second)); + } + + private boolean colStatsAccurate(TableIdentifier identifier) { + return StatsSetupConst.areColumnStatsUptoDate(hmsTable(identifier).getParameters(), "id"); + } + + /** The granularity this table's statistics are written at, which is what a read asks back for. */ + private boolean partitionLevel(Table icebergTable) { + return IcebergTableUtil.isPartitionStats(icebergTable, shell.getHiveConf()); + } + + private List readCurrentColStats(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + return readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); + } + + /** The stored entries at the table's granularity, one per partition or one for the table. */ + private List readColStats(Table icebergTable, long snapshotId) { + if (!partitionLevel(icebergTable)) { + List statsObjs = IcebergColStatsReader.readColStats(icebergTable, snapshotId, null); + if (statsObjs.isEmpty()) { + return List.of(); + } + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, "default", icebergTable.name()); + return List.of(new ColumnStatistics(statsDesc, statsObjs)); + } + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(icebergTable, snapshotId, true); + if (statsFile == null) { + return List.of(); + } + return IcebergColStatsReader.readPartColStats(icebergTable, statsFile, null, null).entrySet().stream() + .map(entry -> { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "default", icebergTable.name()); + statsDesc.setPartName(entry.getKey()); + return new ColumnStatistics(statsDesc, entry.getValue()); + }).toList(); + } + + private boolean hasColStatsForCurrentSnapshot(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + long snapshotId = icebergTable.currentSnapshot().snapshotId(); + return icebergTable.statisticsFiles().stream().anyMatch(statsFile -> statsFile.snapshotId() == snapshotId); + } + + @Test + public void testIncrementalColStatsAfterTruncate() { + // truncate empties the table, so the next increment is the whole table and re-anchors the chain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_truncated"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG " + + "STORED AS PARQUET TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (0), (1), (2)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 0, 2); + + shell.executeStatement("TRUNCATE TABLE " + identifier); + // the ancestor walk stops at the empty snapshot: nothing is served and no stats file is read + Assert.assertTrue(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (10), (11), (12)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 10, 12); + } + + @Test + public void testColStatsAfterPartitionTruncate() { + // a partition truncate clears the accuracy flag like any DML: the pre-truncate file keeps + // serving (the pruner never requests the wiped partition) and ANALYZE recomputes exactly + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_part_truncated"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("TRUNCATE TABLE " + identifier + " PARTITION (p = 'a')"); + Assert.assertFalse(colStatsAccurate(identifier)); + // the emptied partition describes rows it no longer holds, and the one beside it is untouched + Assert.assertEquals(ImmutableMap.of("p=a", false, "p=b", true), + upToDateByPartition(testTables.loadTable(identifier))); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + Assert.assertFalse(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getHighValue()); + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); + } + + + @Test + public void testVoidTransformEvolutionUnifiesPartitionNames() { + // a V1 removal keeps the field as a void transform: a legacy row with a null value and the + // new-spec rows project to the same unified partition tuple, so their statistics merge under + // one name - exactly as Iceberg's own partition statistics unify the rows + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_ambiguous"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, a string, b string) " + + "PARTITIONED BY SPEC (a, b) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='1')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'x', NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (a)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 'x', 'whatever')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj id = colStatsObj(colStats, "a=x/b=" + NULL_PART, "id"); + Assert.assertEquals(1L, id.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testAggrColStatsAfterBucketAndYearEvolutionsFromUnpartitioned() throws Exception { + // unpartitioned history plus two partitioned specs - different bucket widths and a year + // transform - with null and empty-string source values scattered across all three: one ANALYZE + // pass groups every row under the spec that wrote it and names the blobs like the read side + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evolved"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, a string, b date) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (0, 'x', date '2023-03-04'), " + + "(1, '', NULL), (2, NULL, date '2024-06-01')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(8, a))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'x', date '2023-05-05'), " + + "(4, '', date '2023-06-06'), (5, NULL, NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(4, a), year(b))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'x', date '2023-07-07'), " + + "(7, '', date '2024-08-08'), (8, NULL, NULL)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // the null source values produce null partition values; the empty string hashes to a genuine bucket + List statNames = Stream.of( + DummyPartition.VOID, + "a_bucket_8=" + bucket(8, "x"), "a_bucket_8=" + bucket(8, ""), "a_bucket_8=" + NULL_PART, + "a_bucket_4=" + bucket(4, "x") + "/b_year=2023", + "a_bucket_4=" + bucket(4, "") + "/b_year=2024", + "a_bucket_4=" + NULL_PART + "/b_year=" + NULL_PART) + .sorted().toList(); + + Assert.assertEquals(statNames, colStatsPartNames(identifier)); + // ids 0..2 exist only among the unpartitioned rows, 6..8 only in the latest spec's + assertAggrColStatsRange(identifier, "id", statNames, 0, 8); + } + + private static int bucket(int numBuckets, String value) { + return Transforms.bucket(numBuckets).bind(Types.StringType.get()).apply(value); + } + + private void createDatePartitionedTable(TableIdentifier identifier) { + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, d date) " + + "PARTITIONED BY SPEC (year(d)) STORED BY ICEBERG STORED AS PARQUET"); + } + + /** The named column's statistics object within the named partition's blob. */ + private static ColumnStatisticsObj colStatsObj(List colStats, String partName, String colName) { + return colStats.stream() + .filter(stats -> partName.equals(stats.getStatsDesc().getPartName())) + .flatMap(stats -> stats.getStatsObj().stream()) + .filter(obj -> colName.equals(obj.getColName())) + .findFirst().orElseThrow(); + } + + /** The persisted column-statistics blobs' partition names, sorted. */ + private List colStatsPartNames(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + List colStats = + readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); + return colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList(); + } + + /** Asserts a complete aggregation over the given partition names: none missing, min/max spanning. */ + private void assertAggrColStatsRange(TableIdentifier identifier, String column, List statNames, + long lowValue, long highValue) throws Exception { + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable, ImmutableList.of(column), statNames); + Assert.assertEquals(statNames.size(), aggrStats.getPartsFound()); + ColumnStatisticsObj statsObj = aggrStats.getColStats().get(0); + Assert.assertEquals(lowValue, statsObj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(highValue, statsObj.getStatsData().getLongStats().getHighValue()); + } + private void createEvolvedCustomers(TableIdentifier identifier) { shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, @@ -693,6 +2364,58 @@ private void createPartitionedCustomers(TableIdentifier identifier, boolean auto HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); } + @Test + public void testPartitionNameRendersAcrossEvolutionsAndTypes() { + // SELECT PARTITION__NAME renders every row's name under its writing spec - identity with + // characters partitionToPath escapes, a day transform over timestamps, nulls, and the + // unpartitioned history - byte-equal to the names the partition listing produces + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_name_render"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, s string, ts timestamp) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'x=1/y', timestamp '2023-03-04 10:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (s)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 'a b', timestamp '2023-03-04 11:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (day(ts))"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (3, 'c', timestamp '2024-06-01 12:00:00'), (4, NULL, NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(4, s))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'bucketed', timestamp '2025-01-01 00:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (truncate(2, s), month(ts))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'trunc-me', timestamp '2025-02-03 04:05:06')"); + + List rows = shell.executeStatement( + "SELECT id, PARTITION__NAME FROM " + identifier + " ORDER BY id"); + // every row renders its own writing spec's name + Assert.assertEquals(6, rows.size()); + Assert.assertEquals(DummyPartition.VOID, rows.get(0)[1]); + Assert.assertEquals("s=a+b", rows.get(1)[1]); + Assert.assertEquals("ts_day=2024-06-01", rows.get(2)[1]); + Assert.assertEquals("ts_day=" + NULL_PART, rows.get(3)[1]); + Integer bucket = Transforms.bucket(4).bind(Types.StringType.get()).apply("bucketed"); + Assert.assertEquals("s_bucket_4=" + bucket, rows.get(4)[1]); + Assert.assertEquals("s_trunc_2=tr/ts_month=2025-02", rows.get(5)[1]); + Set served = rows.stream().map(r -> String.valueOf(r[1])).collect(Collectors.toSet()); + + // every spec's partitions, not only the latest spec's: rows keep their writing spec's name + Set expected; + try { + expected = storageHandler().getPartitions(hmsTable(identifier), Collections.emptyMap(), false).stream() + .map(Partition::getName) + .collect(Collectors.toSet()); + } catch (SemanticException e) { + throw new RuntimeException(e); + } + // the legacy unpartitioned rows belong to no partition: the synthetic no-partition name + expected.add(DummyPartition.VOID); + Assert.assertEquals(expected, served); + + // the analyzed statistics land under exactly the served names + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(expected.stream().sorted().toList(), colStatsPartNames(identifier)); + } + private org.apache.hadoop.hive.ql.metadata.Table hmsTable(TableIdentifier identifier) { try { return new org.apache.hadoop.hive.ql.metadata.Table( diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java new file mode 100644 index 000000000000..736276f4c9dc --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.iceberg.Files; +import org.apache.iceberg.util.ByteBuffers; +import org.junit.Assert; +import org.junit.Test; + +/** The partition blob frame: what a read fetches and decodes is only what it asked for. */ +public class TestIcebergColStatsFormat { + + @Test + public void rangedReadFetchesOnlyTheAskedColumns() throws Exception { + // 3000 columns also push the header past the probe, so it is completed by a second read + List statsObjs = columns(3000); + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(statsObjs); + int blobLength = blob.remaining(); + File file = blobAt(blob, 123); + + List read = IcebergColStatsReader.readSlices( + Files.localInput(file), 123, blobLength, Set.of("c0", "c1499", "c2999")); + + Assert.assertEquals(List.of("c0", "c1499", "c2999"), + read.stream().map(ColumnStatisticsObj::getColName).toList()); + for (ColumnStatisticsObj obj : read) { + long ordinal = Long.parseLong(obj.getColName().substring(1)); + Assert.assertEquals(ordinal, obj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2 * ordinal, obj.getStatsData().getLongStats().getHighValue()); + } + } + + @Test + public void wholeBlobDecodeAgreesWithTheRangedRead() throws Exception { + List statsObjs = columns(40); + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(statsObjs); + int blobLength = blob.remaining(); + File file = blobAt(blob.duplicate(), 0); + + Set asked = Set.of("c7", "c8", "c31"); + Assert.assertEquals( + IcebergColStatsReader.decodePartitionBlob(blob, asked), + IcebergColStatsReader.readSlices(Files.localInput(file), 0, blobLength, asked)); + } + + @Test + public void unknownFrameReadsAsAbsent() throws Exception { + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(columns(3)); + blob.putInt(0, 999); + int blobLength = blob.remaining(); + File file = blobAt(blob.duplicate(), 0); + + Assert.assertEquals(List.of(), + IcebergColStatsReader.readSlices(Files.localInput(file), 0, blobLength, Set.of("c1"))); + Assert.assertEquals(List.of(), IcebergColStatsReader.decodePartitionBlob(blob, Set.of("c1"))); + } + + private static List columns(int count) { + return IntStream.range(0, count).mapToObj(i -> { + LongColumnStatsData longStats = new LongColumnStatsData(0, i + 1); + longStats.setLowValue(i); + longStats.setHighValue(2L * i); + return new ColumnStatisticsObj("c" + i, "bigint", ColumnStatisticsData.longStats(longStats)); + }).toList(); + } + + /** The blob written at an offset, as it sits inside a Puffin file. */ + private static File blobAt(ByteBuffer blob, int offset) throws Exception { + File file = File.createTempFile("colstats-frame", ".bin"); + file.deleteOnExit(); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(new byte[offset]); + out.write(ByteBuffers.toByteArray(blob)); + } + return file; + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java new file mode 100644 index 000000000000..eaeb66fc0e1c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.Facts; +import org.junit.Test; + +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.MERGE; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.REPLACE; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.SKIP; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.resolve; +import static org.junit.Assert.assertEquals; + +/** Every case of the decision, stated as one, without a session to make it in. */ +public class TestIcebergColStatsWritePolicy { + + @Test + public void analyzeOfTheWholeTableReplacesEveryPartition() { + assertEquals(REPLACE, resolve(partitionLevel().analyze().build())); + } + + @Test + public void analyzeOfNamedPartitionsKeepsTheOthers() { + assertEquals(MERGE, resolve(partitionLevel().analyzePartition().build())); + } + + @Test + public void analyzeIgnoresWhatTheSnapshotItReadsHolds() { + // an ANALYZE inherits the last write's snapshot; what that covered says nothing about its own + assertEquals(REPLACE, resolve(partitionLevel().analyze().holdsOnlyAddedRows().build())); + assertEquals(MERGE, resolve(partitionLevel().analyzePartition().holdsOnlyAddedRows().build())); + } + + @Test + public void analyzeRecomputesEvenWhenTheStoredStatisticsAreCurrent() { + assertEquals(REPLACE, resolve(partitionLevel().analyze().statsAccurate().wroteNoRows().build())); + } + + @Test + public void aWriteThatProducedEveryRowReplaces() { + assertEquals(REPLACE, resolve(partitionLevel().holdsOnlyAddedRows().build())); + assertEquals(REPLACE, resolve(partitionLevel().emptySnapshot().build())); + } + + @Test + public void aWriteThatReplacedSomePartitionsStandsForThem() { + assertEquals(MERGE, resolve(partitionLevel().replacePartitions().build())); + // a plain insert only added to partitions, and rows of part of a partition describe none of it + assertEquals(SKIP, resolve(partitionLevel().build())); + } + + @Test + public void aWriteWithNothingToRecordLeavesThePartitionStatisticsAlone() { + assertEquals(SKIP, resolve(partitionLevel().wroteNoRows().build())); + assertEquals(SKIP, resolve(partitionLevel().statsAccurate().build())); + } + + @Test + public void tableWideStatisticsOfATableThatKeepsThemPerPartitionGoNowhere() { + assertEquals(SKIP, resolve(tableWideStats().keepsStatsPerPartition().analyze().build())); + } + + @Test + public void onlyAWholeTableCompactionRefreshesStaleStatistics() { + assertEquals(REPLACE, resolve(tableWideStats().fullTableMajorCompaction().build())); + assertEquals(SKIP, resolve(tableWideStats().compaction().build())); + assertEquals(SKIP, resolve(tableWideStats().fullTableMajorCompaction().statsAccurate().build())); + // a partitioned table's compaction never rewrites the whole table, so it cannot refresh it + assertEquals(SKIP, resolve(tableWideStats().majorCompaction().build())); + } + + @Test + public void aCompactionSubstitutesOnlyThePartitionsItReadWhole() { + // the numbers of a partition it rewrote entirely stand for it, whatever it held before + assertEquals(MERGE, resolve(partitionLevel().majorCompaction().singlePartitionRewrite().build())); + assertEquals(MERGE, + resolve(partitionLevel().majorCompaction().singlePartitionRewrite().statsAccurate().build())); + // one that skipped files by size measured part of a partition, which describes none of it + assertEquals(SKIP, resolve(partitionLevel().compaction().singlePartitionRewrite().build())); + // and one clearing an older spec rewrites into partitions holding rows it never read + assertEquals(SKIP, resolve(partitionLevel().majorCompaction().build())); + assertEquals(SKIP, resolve(partitionLevel().compaction().build())); + } + + @Test + public void tableWideAnalyzeReplaces() { + assertEquals(REPLACE, resolve(tableWideStats().analyze().build())); + } + + @Test + public void anEmptiedTableLosesTheStatisticsOfTheRowsItHeld() { + assertEquals(REPLACE, resolve(tableWideStats().emptySnapshot().wroteNoRows().build())); + } + + @Test + public void anInsertAddsItsRowsToTheStoredStatistics() { + assertEquals(MERGE, resolve(tableWideStats().build())); + } + + @Test + public void anOverwriteOfSomePartitionsCannotDescribeTheTable() { + assertEquals(SKIP, resolve(tableWideStats().replacePartitions().partitioned().build())); + assertEquals(REPLACE, resolve(tableWideStats().replacePartitions().build())); + } + + /** A write of per-partition statistics that neither covered the table nor left it empty. */ + private static Builder partitionLevel() { + return new Builder(); + } + + /** A write of statistics describing the table as a whole. */ + private static Builder tableWideStats() { + return new Builder().tableWideStats(); + } + + /** Names each fact, so a case reads as the statement it stands for. */ + private static final class Builder { + private boolean tableWideStats; + private boolean keepsStatsPerPartition; + private boolean analyze; + private boolean analyzePartition; + private boolean compaction; + private boolean majorCompaction; + private boolean fullTableRewrite; + private boolean singlePartitionRewrite; + private boolean holdsOnlyAddedRows; + private boolean emptySnapshot; + private boolean wroteNoRows; + private boolean replacePartitions; + private boolean partitioned; + private boolean statsAccurate; + + private Builder tableWideStats() { + tableWideStats = true; + return this; + } + + private Builder keepsStatsPerPartition() { + keepsStatsPerPartition = true; + return this; + } + + private Builder analyze() { + analyze = true; + return this; + } + + private Builder analyzePartition() { + analyzePartition = true; + return analyze(); + } + + private Builder compaction() { + compaction = true; + return this; + } + + /** A compaction that read every file of what it was pointed at. */ + private Builder majorCompaction() { + majorCompaction = true; + return compaction(); + } + + /** A major compaction pointed at the whole table, which only an unpartitioned one is. */ + private Builder fullTableMajorCompaction() { + fullTableRewrite = true; + return majorCompaction(); + } + + /** A compaction pointed at one named partition, rather than at what an older spec left. */ + private Builder singlePartitionRewrite() { + singlePartitionRewrite = true; + return compaction(); + } + + private Builder holdsOnlyAddedRows() { + holdsOnlyAddedRows = true; + return this; + } + + private Builder emptySnapshot() { + emptySnapshot = true; + return this; + } + + private Builder wroteNoRows() { + wroteNoRows = true; + return this; + } + + private Builder replacePartitions() { + replacePartitions = true; + return this; + } + + private Builder partitioned() { + partitioned = true; + return this; + } + + private Builder statsAccurate() { + statsAccurate = true; + return this; + } + + private Facts build() { + return new Facts(tableWideStats, keepsStatsPerPartition, partitioned, () -> statsAccurate, + analyze, analyzePartition, compaction, majorCompaction, fullTableRewrite, + singlePartitionRewrite, holdsOnlyAddedRows, emptySnapshot, wroteNoRows, replacePartitions); + } + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java index 2a8b0bdc6ed1..28487b715ff4 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java @@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -109,9 +110,10 @@ public void testRowIterator() throws Exception { List records = TestHelper.generateRandomRecords(allSchema, 10, 0L); Table table = testTables.createTable(shell, "temptable", allSchema, fileFormat, records); - // Identify data file location - expected to be 1 file exactly - Path dataFilePath = new Path(Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) - .files().iterator()).get(0).file().path().toString()); + // Identify the scan task - expected to be 1 file exactly + FileScanTask fileScanTask = Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) + .files().iterator()).get(0); + Path dataFilePath = new Path(fileScanTask.file().path().toString()); // Generate a mock vectorized read job JobConf jobConf = prepareMockJob(allSchema, dataFilePath); @@ -122,7 +124,7 @@ public void testRowIterator() throws Exception { inputFormat.getRecordReader(new FileSplit(dataFilePath, 0L, Long.MAX_VALUE, new String[]{}), jobConf, new MockReporter()); HiveBatchIterator hiveBatchIterator = new HiveBatchIterator( - internalVectorizedRecordReader, jobConf, null, null, null); + internalVectorizedRecordReader, jobConf, null, null, fileScanTask); // Expected to be one batch exactly HiveBatchContext hiveBatchContext = hiveBatchIterator.next(); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java index 36ff67cb2068..1f5e8cfa7a04 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java @@ -88,7 +88,8 @@ private static List deleteRecords(Table table, Set idsTo continue; } - GenericRecord deleteRecord = GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns())); + GenericRecord deleteRecord = + GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns(), false)); int specId = (Integer) record.getField(MetadataColumns.SPEC_ID.name()); deleteRecord.setField(MetadataColumns.SPEC_ID.name(), specId); PartitionKey partitionKey = new PartitionKey(table.specs().get(specId), table.schema()); @@ -96,8 +97,6 @@ private static List deleteRecords(Table table, Set idsTo deleteRecord.setField(MetadataColumns.PARTITION_COLUMN_NAME, partitionKey); deleteRecord.setField(MetadataColumns.FILE_PATH.name(), record.getField(MetadataColumns.FILE_PATH.name())); deleteRecord.setField(MetadataColumns.ROW_POSITION.name(), record.getField(MetadataColumns.ROW_POSITION.name())); - deleteRecord.setField(IcebergAcidUtil.PARTITION_PROJECTION.name(), - IcebergAcidUtil.getSerializedPartitionKey(partitionKey, table.spec())); SCHEMA.columns().forEach(field -> deleteRecord.setField(field.name(), record.getField(field.name()))); deleteRecords.add(deleteRecord); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q new file mode 100644 index 000000000000..66b258bafde7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q @@ -0,0 +1,81 @@ +-- what ANALYZE stores for a table whose partitioning changed under it +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the inserts must leave the stored statistics alone for what happens to them to be visible +set hive.stats.column.autogather=false; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +-- these rows belong to the synthetic partition, the one an unpartitioned spec names +insert into ice_evo values (1, 'a'), (100, 'b'); + +alter table ice_evo set partition spec (p); + +insert into ice_evo values (5, 'a'); + +-- a full table ANALYZE reads every partition of every spec the table holds +analyze table ice_evo compute statistics for columns; + +explain select id from ice_evo where id > 0; + +select min(id), max(id) from ice_evo; + +-- a write reaching one partition leaves the others describing themselves +insert into ice_evo values (7, 'a'); + +explain select id from ice_evo where id > 0; + +select min(id), max(id) from ice_evo; + +-- naming that partition measures it again; the ones it never named are carried +analyze table ice_evo partition (p='a') compute statistics for columns; + +explain select id from ice_evo where id > 0; + +select min(id), max(id) from ice_evo; + +drop table ice_evo; + +-- a table described in full, then partitioned differently and written to: what the new spec's +-- partitions hold is not described until an ANALYZE reads them +create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_evo2 values (1, 'a'), (7, 'b'); +analyze table ice_evo2 compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +select min(id), max(id) from ice_evo2; + +alter table ice_evo2 set partition spec (p, truncate(1, p)); + +insert into ice_evo2 values (9, 'a'); + +explain select id from ice_evo2 where id > 0; + +select min(id), max(id) from ice_evo2; + +-- what the user sees before reaching for a partition scoped ANALYZE: partitions of both specs +show partitions ice_evo2; + +-- naming a partition measures the one the current spec writes today; the older spec's p=a keeps +-- the statistics it already had, which no write since could have changed +analyze table ice_evo2 partition (p='a') compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +select min(id), max(id) from ice_evo2; + +analyze table ice_evo2 compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +select min(id), max(id) from ice_evo2; + +drop table ice_evo2; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q new file mode 100644 index 000000000000..e5a2cb87122e --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q @@ -0,0 +1,37 @@ +-- SORT_QUERY_RESULTS +set hive.explain.user=false; +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.fetch.task.conversion=none; + +create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2'); + +insert into ice_p values (1,'a'),(2,'a'),(7,'b'); + +-- 1) partition-level statistics for the current snapshot +set hive.iceberg.stats.collect.partlevel=true; +analyze table ice_p compute statistics for columns; +describe formatted ice_p id; +explain select * from ice_p where p='a'; + +-- 2) a new snapshot, then table-level statistics for it +insert into ice_p values (9,'c'); +set hive.iceberg.stats.collect.partlevel=false; +analyze table ice_p compute statistics for columns; +describe formatted ice_p id; +-- table-level statistics serve the pruned set +explain select * from ice_p where p='a'; + +-- 3) toggle back: the current snapshot's file is table-level shaped, so the partition-level +-- read falls back to the older partition-level file of the ancestor snapshot +set hive.iceberg.stats.collect.partlevel=true; +explain select * from ice_p where p='a'; +describe formatted ice_p id; + +-- 4) recompute at the current snapshot with the flag on +analyze table ice_p compute statistics for columns; +explain select * from ice_p where p='a'; +describe formatted ice_p id; + +drop table ice_p; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q new file mode 100644 index 000000000000..66eccc02efab --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q @@ -0,0 +1,59 @@ +-- what a write leaves of the column statistics stored before it, at each granularity +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the writes below must leave the stored statistics alone for what happens to them to be visible +set hive.stats.column.autogather=false; + +-- an unpartitioned table keeps one set for the whole of it, and a write leaves none of it standing +create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_unpart values (1, 'a'), (2, 'b'); +analyze table ice_stale_unpart compute statistics for columns; + +explain select id from ice_stale_unpart where id > 0; + +insert into ice_stale_unpart values (3, 'c'); + +explain select id from ice_stale_unpart where id > 0; + +drop table ice_stale_unpart; + +-- a partitioned table asked for whole-table statistics keeps one set too, on the same terms +set hive.iceberg.stats.collect.partlevel=false; + +create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c'); +analyze table ice_stale_tbllevel compute statistics for columns; + +explain select id from ice_stale_tbllevel where id > 0; + +insert into ice_stale_tbllevel values (9, 'a'); + +explain select id from ice_stale_tbllevel where id > 0; + +drop table ice_stale_tbllevel; + +-- kept per partition, only the partitions a write reached stop describing themselves, and what +-- the rest still describe is a part of what the scan reads +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd'); +analyze table ice_stale_partlevel compute statistics for columns; + +explain select id from ice_stale_partlevel where id > 0; + +insert into ice_stale_partlevel values (9, 'a'), (9, 'b'); + +explain select id from ice_stale_partlevel where id > 0; + +drop table ice_stale_partlevel; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q new file mode 100644 index 000000000000..0abb64656544 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q @@ -0,0 +1,51 @@ +-- what each kind of write leaves describing the table +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); +insert into ice_src values (1, 'a'), (7, 'b'); + +-- a CREATE TABLE AS reads every row it writes, so it describes all of them +create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src; + +explain select id from ice_ctas where id > 0; + +drop table ice_ctas; + +-- an insert gathers as it writes, so an unpartitioned table stays described +create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_unpart_w values (1, 'a'), (7, 'b'); +analyze table ice_unpart_w compute statistics for columns; + +explain select id from ice_unpart_w where id > 0; + +insert into ice_unpart_w values (9, 'c'); + +explain select id from ice_unpart_w where id > 0; + +drop table ice_unpart_w; + +-- an overwrite of the whole table replaces the rows and what described them alike +create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_iow values (1, 'a'), (7, 'b'); +analyze table ice_iow compute statistics for columns; + +explain select id from ice_iow where id > 0; + +insert overwrite table ice_iow select id + 100, p from ice_src; + +explain select id from ice_iow where id > 0; + +drop table ice_iow; +drop table ice_src; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q new file mode 100644 index 000000000000..8fa963997b79 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q @@ -0,0 +1,55 @@ +-- what a compaction leaves behind for a table that had no column statistics +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the inserts must gather nothing, so that what the compaction gathers is what shows +set hive.stats.column.autogather=false; +set hive.iceberg.stats.collect.partlevel=true; + +-- a major compaction of an unpartitioned table reads every row it holds, so what it measures +-- describes the whole of it +create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500'); + +insert into ice_comp_unpart values (1, 'a'); +insert into ice_comp_unpart values (2, 'a'); +insert into ice_comp_unpart values (3, 'a'); +insert into ice_comp_unpart values (7, 'b'); + +explain select id from ice_comp_unpart where id > 0; + +alter table ice_comp_unpart COMPACT 'major' and wait; + +explain select id from ice_comp_unpart where id > 0; + +select min(id), max(id) from ice_comp_unpart; + +drop table ice_comp_unpart; + +-- a major compaction of one partition reads every row of that partition, and none of the others, +-- so it describes that partition alone +create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true'); + +insert into ice_comp values (1, 'a'); +insert into ice_comp values (2, 'a'); +insert into ice_comp values (3, 'a'); +insert into ice_comp values (4, 'a'); +insert into ice_comp values (7, 'b'); + +explain select id from ice_comp where id > 0; + +alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait; + +explain select id from ice_comp where id > 0; + +select min(id), max(id) from ice_comp; + +drop table ice_comp; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q new file mode 100644 index 000000000000..147e373c18b2 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -0,0 +1,152 @@ +--! qt:replace:/(\s+Statistics\: Num rows\: \d+ Data size\:\s+)\S+(\s+Basic stats\: \S+ Column stats\: \S+)/$1#Masked#$2/ + +-- Column statistics kept per partition answer a query over the partitions it pruned to, and stop +-- answering only for the partitions a later write reached. + +set hive.explain.user=false; +set hive.compute.query.using.stats=true; +set hive.fetch.task.conversion=none; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c'); +analyze table ice_part_stats compute statistics for columns; + +-- answered from the statistics of the pruned partition alone +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- a write that reaches p=a only +insert into ice_part_stats values (11, 'a'); + +-- a scan spanning the partition the write reached and one it did not has statistics for only part +-- of what it reads, which is what PARTIAL says +explain +select id from ice_part_stats where p in ('a', 'b'); + +-- p=a describes itself no longer, so the query has to read it +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- the partitions that write never touched still answer from their statistics +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- an ANALYZE naming the written partition measures it again, and it answers from statistics once +-- more while the partitions carried across that ANALYZE keep the numbers they were computed with +analyze table ice_part_stats partition (p = 'a') compute statistics for columns; + +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- count(col) needs a row count as well as the column's null count, and a handler keeps no +-- partition parameters to read one from: it is asked of the table for the pruned partitions +explain +select count(id) from ice_part_stats where p = 'b'; + +select count(id) from ice_part_stats where p = 'b'; + +-- a query spanning a written and an untouched partition cannot be answered from a subset +explain +select max(id) from ice_part_stats where p in ('a', 'b'); + +select max(id) from ice_part_stats where p in ('a', 'b'); + +drop table ice_part_stats; + +-- an unpartitioned table keeps its statistics in the same file, which the metastore never holds: +-- reaching them takes the handler, and only the accuracy check stands between a query and stale ones +create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_unpart values (1), (5), (9); +analyze table ice_unpart compute statistics for columns; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- an incremental gather keeps them describing the table, so it still answers +insert into ice_unpart values (11); + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- a write that records nothing, as another engine's would, leaves them behind: only the accuracy +-- check stands between the query and a value the table no longer holds +set hive.stats.autogather=false; +insert into ice_unpart values (20); +set hive.stats.autogather=true; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +drop table ice_unpart; + +-- statistics kept for the table as a whole describe no partition in particular, so a query over +-- one of them cannot be answered from them however fresh they are +set hive.iceberg.stats.collect.partlevel=false; + +create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b'); +analyze table ice_tbl_level compute statistics for columns; + +-- a scan reading every partition reads the whole table, which is what they do describe +explain +select max(id) from ice_tbl_level; + +select max(id) from ice_tbl_level; + +explain +select count(id) from ice_tbl_level; + +select count(id) from ice_tbl_level; + +explain +select max(id) from ice_tbl_level where p = 'a'; + +select max(id) from ice_tbl_level where p = 'a'; + +drop table ice_tbl_level; + +set hive.iceberg.stats.collect.partlevel=true; + +-- with the statistics kept by the metastore there are no per-partition numbers to answer from +set hive.iceberg.stats.source=metastore; + +create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b'); + +explain +select max(id) from ice_part_stats_hms where p = 'a'; + +select max(id) from ice_part_stats_hms where p = 'a'; + +drop table ice_part_stats_hms; diff --git a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out index 23b2d41d3dd0..e0767e06c45f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out @@ -45,13 +45,13 @@ Stage-0 Filter Operator [FIL_17] (rows=2 width=265) predicate:(id is not null and part is not null) TableScan [TS_3] (rows=2 width=265) - default@tbl,tbl2,Tbl:COMPLETE,Col:COMPLETE,Output:["foid","part","id"] + default@tbl,tbl2,Tbl:COMPLETE,Col:PARTIAL,Output:["foid","part","id"] <-Select Operator [SEL_21] (rows=2 width=265) Output:["_col0","_col1","_col2"] Filter Operator [FIL_20] (rows=2 width=265) predicate:(id is not null and part is not null) TableScan [TS_0] (rows=2 width=265) - default@tbl,tbl,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] + default@tbl,tbl,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] PREHOOK: query: SELECT * FROM tbl JOIN tbl tbl2 ON tbl.id = tbl2.id AND tbl.part = tbl2.part PREHOOK: type: QUERY diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out index ef5c4410fbb6..50f7b4b6059b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out @@ -321,25 +321,9 @@ POSTHOOK: Input: default@tbl_ice_puffin POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. -Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Stage-0 Fetch Operator - limit:-1 - Stage-1 - Reducer 2 vectorized - File Output Operator [FS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_9] - Group By Operator [GBY_8] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(a)","max(c)"] - Select Operator [SEL_7] (rows=5 width=8) - Output:["a","c"] - TableScan [TS_0] (rows=5 width=8) - default@tbl_ice_puffin,tbl_ice_puffin,Tbl:COMPLETE,Col:COMPLETE,Output:["a","c"] + limit:1 PREHOOK: query: desc formatted tbl_ice_puffin C PREHOOK: type: DESCTABLE diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index a531c87a685c..810776ea1de3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -121,77 +121,77 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(c) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Map 22 @@ -199,58 +199,58 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Group By Operator keys: a (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(a) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 10 @@ -262,11 +262,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 112 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 112 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -277,13 +277,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 190 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 190 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -294,23 +294,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -330,26 +330,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -358,15 +358,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1206 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -375,13 +375,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1326 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1326 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -390,15 +390,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 1404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 1404 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -407,25 +407,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 18 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -444,21 +444,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 772 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 772 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 19 Execution mode: vectorized Reduce Operator Tree: @@ -466,16 +466,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -485,13 +485,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -499,31 +499,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 21 Execution mode: vectorized @@ -532,11 +532,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 23 Execution mode: vectorized @@ -545,11 +545,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 24 Execution mode: vectorized @@ -558,31 +558,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 25 Execution mode: vectorized @@ -591,11 +591,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 26 Execution mode: vectorized @@ -604,11 +604,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -618,19 +618,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -639,13 +639,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 1505 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 1505 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -654,26 +654,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -682,29 +682,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1494 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -719,13 +719,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 102 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 102 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Union 7 Vertex: Union 7 @@ -795,7 +795,7 @@ POSTHOOK: query: insert into tbl_ice_other values (10, 'ten'), (333, 'hundred') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice_other -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -817,17 +817,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -839,8 +838,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -848,47 +847,47 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -909,19 +908,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -930,20 +916,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -963,44 +949,44 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -1010,13 +996,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1025,18 +1011,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1044,7 +1030,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1053,35 +1039,21 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1089,22 +1061,22 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) - Reducer 8 + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1123,22 +1095,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 9 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1147,54 +1120,67 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1215,7 +1201,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1259,7 +1245,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_standard_other POSTHOOK: Lineage: tbl_standard_other.a SCRIPT [] POSTHOOK: Lineage: tbl_standard_other.b SCRIPT [] -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1281,17 +1267,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 12 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1303,8 +1288,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1312,47 +1297,47 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -1373,25 +1358,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -1400,20 +1366,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1433,38 +1399,38 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1474,13 +1440,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1489,18 +1455,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1508,7 +1474,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1517,35 +1483,21 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 646 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1553,22 +1505,22 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) - Reducer 8 + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1587,22 +1539,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - Reducer 9 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 646 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1611,62 +1564,81 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col1 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: boolean) Union 5 Vertex: Union 5 @@ -1687,7 +1659,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[155][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[153][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out index 9a5350c2e0f2..55cd325ea776 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out @@ -54,16 +54,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -92,15 +92,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -156,9 +156,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -177,17 +177,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -296,129 +296,129 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: a (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: c (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Reducer 10 @@ -430,13 +430,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -447,23 +447,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -483,26 +483,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -511,15 +511,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -528,13 +528,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -543,15 +543,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -560,25 +560,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -597,17 +597,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1431 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1455 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -620,31 +620,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 19 Execution mode: vectorized @@ -653,11 +653,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -667,13 +667,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -681,11 +681,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 21 Execution mode: vectorized @@ -694,11 +694,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 22 Execution mode: vectorized @@ -707,31 +707,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 23 Execution mode: vectorized @@ -740,16 +740,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 24 Execution mode: vectorized @@ -758,11 +758,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -772,19 +772,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -793,13 +793,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -808,26 +808,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -836,15 +836,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -859,13 +859,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 9 Reduce Operator Tree: @@ -876,11 +876,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Union 7 Vertex: Union 7 @@ -993,8 +993,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1002,13 +1002,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1016,7 +1016,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1151,13 +1151,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1166,18 +1166,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1185,7 +1185,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1194,11 +1194,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1216,7 +1216,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1224,13 +1224,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1255,8 +1255,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1431,8 +1431,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1440,13 +1440,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1454,7 +1454,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1583,13 +1583,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1598,18 +1598,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1617,7 +1617,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1626,11 +1626,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1648,7 +1648,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1656,13 +1656,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1687,8 +1687,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out index aed511e4b8fe..d8f7ecf10e8d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out @@ -85,14 +85,14 @@ Stage-4 File Output Operator [FS_46] table:{"name:":"default.ice01"} Select Operator [SEL_44] (rows=7 width=206) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_43] (rows=7 width=206) - Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_59] PartitionCols:_col4 Select Operator [SEL_56] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_53] (rows=7 width=188) predicate:((((id = 2) or (id > 4)) is null or (id < 2) or ((id > 2) and (id <= 4))) and FILE__PATH is not null) TableScan [TS_0] (rows=7 width=188) @@ -120,13 +120,13 @@ Stage-4 File Output Operator [FS_70] table:{"name:":"default.ice01"} Select Operator [SEL_69] (rows=3 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_68] (rows=3 width=188) predicate:(row_number_window_0 = 1) PTF Operator [PTF_67] (rows=7 width=188) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] Select Operator [SEL_66] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_58] PartitionCols:FILE__PATH diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out index e6227f4cae64..b980414bdaa6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out @@ -306,17 +306,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -362,17 +362,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -381,10 +381,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -435,17 +435,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -475,17 +475,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -494,10 +494,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -566,17 +566,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -660,13 +660,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -675,17 +675,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -694,10 +694,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -757,17 +757,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -819,13 +819,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -834,17 +834,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -853,10 +853,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -922,17 +922,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -994,17 +994,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1013,10 +1013,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1067,17 +1067,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1107,17 +1107,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1126,10 +1126,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1191,17 +1191,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1247,17 +1247,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1266,10 +1266,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1320,17 +1320,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1360,17 +1360,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1379,10 +1379,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1442,17 +1442,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1501,17 +1501,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1520,10 +1520,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1574,17 +1574,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1617,17 +1617,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1636,10 +1636,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1690,17 +1690,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1762,17 +1762,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1781,10 +1781,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1840,20 +1840,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -1937,13 +1937,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1952,17 +1952,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -1971,10 +1971,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2034,20 +2034,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -2131,13 +2131,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -2146,17 +2146,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -2165,10 +2165,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2223,17 +2223,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -2275,17 +2275,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -2294,10 +2294,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2426,17 +2426,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -2479,19 +2479,19 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Execution mode: vectorized Reduce Operator Tree: @@ -2499,10 +2499,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2633,25 +2633,25 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: ds (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 6 Map Operator Tree: @@ -2692,13 +2692,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -2708,10 +2708,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2723,13 +2723,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -2859,11 +2859,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -2872,17 +2872,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -2928,10 +2928,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2999,11 +2999,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3013,7 +3013,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3022,17 +3022,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3114,10 +3114,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3183,11 +3183,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3196,17 +3196,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3268,10 +3268,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3333,11 +3333,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3346,17 +3346,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3402,10 +3402,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3456,11 +3456,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3469,17 +3469,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3528,10 +3528,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3599,14 +3599,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3616,7 +3616,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3625,17 +3625,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3717,10 +3717,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3779,14 +3779,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3796,7 +3796,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3805,17 +3805,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3897,10 +3897,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3955,11 +3955,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Left Semi Join 0 to 1 @@ -3969,19 +3969,19 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Union 5 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 3 Map Operator Tree: @@ -4022,10 +4022,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4181,11 +4181,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -4194,17 +4194,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2000 Data size: 16000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 16000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Map 3 Map Operator Tree: @@ -4263,10 +4263,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out index cd09db327734..eedb29f7095a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out @@ -132,7 +132,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -148,7 +148,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -156,15 +156,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -226,7 +225,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -238,21 +237,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -315,7 +313,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -331,7 +329,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col2","_col1","iceberg_bucket(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1, iceberg_bucket(_col2, 3) Select Operator [SEL_12] (rows=22 width=94) Output:["_col0","_col1","_col2"] @@ -339,15 +337,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=11 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_19] (rows=11 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=11 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + Select Operator [SEL_20] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_19] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_14] (rows=22 width=94) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_12] @@ -450,7 +447,7 @@ POSTHOOK: Output: default@tbl_target_mixed Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -462,11 +459,11 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=3 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=3 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_19] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_14] table:{"name:":"default.tbl_target_mixed"} Select Operator [SEL_13] (rows=4 width=99) @@ -475,10 +472,9 @@ Stage-3 predicate:(b = 'EUR') TableScan [TS_0] (rows=22 width=94) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=3 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_15] (rows=4 width=99) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_13] @@ -502,7 +498,7 @@ POSTHOOK: Output: default@tbl_target_mixed Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -514,11 +510,11 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_19] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_14] table:{"name:":"default.tbl_target_mixed"} Select Operator [SEL_13] (rows=1 width=99) @@ -527,10 +523,9 @@ Stage-3 predicate:((c = 100L) and (b = 'USD')) TableScan [TS_0] (rows=22 width=94) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=1 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_15] (rows=1 width=99) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_13] @@ -677,7 +672,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -693,7 +688,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col1, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -701,15 +696,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=5 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=5 width=516) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=5 width=584) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(ccy, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -772,7 +766,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -788,7 +782,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col0, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col0, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -796,15 +790,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=21 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=21 width=516) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=21 width=584) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(id)","max(id)","count(1)","count(id)","compute_bit_vector_hll(id)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(id, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(id)","max(id)","count(1)","count(id)","compute_bit_vector_hll(id)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["ccy","id"] Please refer to the previous Select Operator [SEL_12] @@ -867,7 +860,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -883,7 +876,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col1, 2) Select Operator [SEL_12] (rows=22 width=11) Output:["_col0","_col1"] @@ -891,15 +884,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","c"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=8 width=752) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=8 width=520) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=8 width=520) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(ccy, 2) + Select Operator [SEL_20] (rows=1 width=528) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=336) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=336) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=11) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -962,7 +954,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -978,7 +970,7 @@ Stage-3 Select Operator [SEL_18] Output:["_col0","_col1","_col2","iceberg_truncate(_col1, 2)","iceberg_truncate(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_14] + PARTITION_ONLY_SHUFFLE [RS_14] PartitionCols:iceberg_truncate(_col1, 2), iceberg_truncate(_col2, 3) Select Operator [SEL_13] (rows=22 width=199) Output:["_col0","_col1","_col2"] @@ -986,15 +978,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_22] - Select Operator [SEL_21] (rows=5 width=1421) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_20] (rows=5 width=1076) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=5 width=1144) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(b))","avg(COALESCE(length(b),0))","count(b)","compute_bit_vector_hll(b)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(b, 2), iceberg_truncate(ccy, 3) + Select Operator [SEL_21] (rows=1 width=1005) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_20] (rows=1 width=708) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=776) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(b))","avg(COALESCE(length(b),0))","count(b)","compute_bit_vector_hll(b)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_15] (rows=22 width=199) Output:["a","b","ccy"] Please refer to the previous Select Operator [SEL_13] @@ -1063,7 +1054,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1079,7 +1070,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_year(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_year(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1091,15 +1082,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_year(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1149,7 +1139,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1165,7 +1155,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_year(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_year(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1177,15 +1167,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_year(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1235,7 +1224,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1251,7 +1240,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_month(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_month(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1263,15 +1252,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_month(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1321,7 +1309,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1337,7 +1325,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_month(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_month(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1349,15 +1337,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_month(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1407,7 +1394,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1423,7 +1410,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_day(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_day(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1435,15 +1422,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_day(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1493,7 +1479,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1509,7 +1495,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_day(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_day(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1521,15 +1507,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_day(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1579,7 +1564,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1595,7 +1580,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_hour(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_hour(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1607,15 +1592,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_hour(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1650,7 +1634,7 @@ POSTHOOK: Output: default@tbl_target_identity Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1662,21 +1646,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_identity"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1692,7 +1675,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1704,21 +1687,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1735,7 +1717,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1751,7 +1733,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1759,15 +1741,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1784,7 +1765,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1800,7 +1781,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 3) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1808,15 +1789,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1833,7 +1813,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1849,7 +1829,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1857,15 +1837,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1882,7 +1861,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1898,7 +1877,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 3) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1906,15 +1885,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1930,7 +1908,7 @@ POSTHOOK: Output: default@tbl_target_identity Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1942,21 +1920,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_identity"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1972,7 +1949,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1984,21 +1961,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -2031,7 +2007,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -2047,7 +2023,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -2055,15 +2031,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=2 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=2 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=3 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out index 0ace027030fa..fba84461503d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out @@ -80,20 +80,20 @@ STAGE PLANS: TableScan alias: srcpart_date_n7 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_n3_key1_min) AND DynamicValue(RS_7_srcpart_small_n3_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_n3_key1_bloom_filter))) (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -138,17 +138,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 126 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2090 Data size: 365523 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -157,10 +157,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out new file mode 100644 index 000000000000..38460ab27d23 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out @@ -0,0 +1,555 @@ +PREHOOK: query: create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo +POSTHOOK: query: create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo +PREHOOK: query: insert into ice_evo values (1, 'a'), (100, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (1, 'a'), (100, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: alter table ice_evo set partition spec (p) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_evo +POSTHOOK: query: alter table ice_evo set partition spec (p) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +PREHOOK: query: insert into ice_evo values (5, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (5, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: analyze table ice_evo compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: default@ice_evo +PREHOOK: Output: default@ice_evo@__NO_PARTITION__ +PREHOOK: Output: default@ice_evo@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +POSTHOOK: Output: default@ice_evo@__NO_PARTITION__ +POSTHOOK: Output: default@ice_evo@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 100 +PREHOOK: query: insert into ice_evo values (7, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (7, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 100 +PREHOOK: query: analyze table ice_evo partition (p='a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: default@ice_evo +PREHOOK: Output: default@ice_evo@__NO_PARTITION__ +PREHOOK: Output: default@ice_evo@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo partition (p='a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +POSTHOOK: Output: default@ice_evo@__NO_PARTITION__ +POSTHOOK: Output: default@ice_evo@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 100 +PREHOOK: query: drop table ice_evo +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo +POSTHOOK: query: drop table ice_evo +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo +PREHOOK: query: create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: insert into ice_evo2 values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: insert into ice_evo2 values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: analyze table ice_evo2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 7 +PREHOOK: query: alter table ice_evo2 set partition spec (p, truncate(1, p)) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_evo2 +POSTHOOK: query: alter table ice_evo2 set partition spec (p, truncate(1, p)) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: insert into ice_evo2 values (9, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: insert into ice_evo2 values (9, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 +PREHOOK: query: show partitions ice_evo2 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@ice_evo2 +POSTHOOK: query: show partitions ice_evo2 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@ice_evo2 +p=a +p=a/p_trunc_1=a +p=b +PREHOOK: query: analyze table ice_evo2 partition (p='a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 partition (p='a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 +PREHOOK: query: analyze table ice_evo2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +PREHOOK: Output: default@ice_evo2@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +POSTHOOK: Output: default@ice_evo2@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 +PREHOOK: query: drop table ice_evo2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: drop table ice_evo2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out new file mode 100644 index 000000000000..f8711b175f14 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out @@ -0,0 +1,326 @@ +PREHOOK: query: create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_p +POSTHOOK: query: create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_p +PREHOOK: query: insert into ice_p values (1,'a'),(2,'a'),(7,'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_p +POSTHOOK: query: insert into ice_p values (1,'a'),(2,'a'),(7,'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_p +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 7 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_p values (9,'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_p +POSTHOOK: query: insert into ice_p values (9,'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_p +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: default@ice_p@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: default@ice_p@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 9 +num_nulls 0 +distinct_count 4 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 7 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"p\":\"true\"}} +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: default@ice_p@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: default@ice_p@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 9 +num_nulls 0 +distinct_count 4 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: drop table ice_p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_p +POSTHOOK: query: drop table ice_p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_p diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out new file mode 100644 index 000000000000..76cedce6ceca --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out @@ -0,0 +1,426 @@ +PREHOOK: query: create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: insert into ice_stale_unpart values (1, 'a'), (2, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: insert into ice_stale_unpart values (1, 'a'), (2, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: analyze table ice_stale_unpart compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_unpart compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_unpart values (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: insert into ice_stale_unpart values (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: explain select id from ice_stale_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: drop table ice_stale_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: analyze table ice_stale_tbllevel compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: default@ice_stale_tbllevel +PREHOOK: Output: default@ice_stale_tbllevel@p=a +PREHOOK: Output: default@ice_stale_tbllevel@p=b +PREHOOK: Output: default@ice_stale_tbllevel@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_tbllevel compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: Output: default@ice_stale_tbllevel@p=a +POSTHOOK: Output: default@ice_stale_tbllevel@p=b +POSTHOOK: Output: default@ice_stale_tbllevel@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_tbllevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_tbllevel values (9, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: insert into ice_stale_tbllevel values (9, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_tbllevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_tbllevel +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: drop table ice_stale_tbllevel +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: analyze table ice_stale_partlevel compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: default@ice_stale_partlevel +PREHOOK: Output: default@ice_stale_partlevel@p=a +PREHOOK: Output: default@ice_stale_partlevel@p=b +PREHOOK: Output: default@ice_stale_partlevel@p=c +PREHOOK: Output: default@ice_stale_partlevel@p=d +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_partlevel compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: default@ice_stale_partlevel +POSTHOOK: Output: default@ice_stale_partlevel@p=a +POSTHOOK: Output: default@ice_stale_partlevel@p=b +POSTHOOK: Output: default@ice_stale_partlevel@p=c +POSTHOOK: Output: default@ice_stale_partlevel@p=d +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_partlevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_partlevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_partlevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_partlevel values (9, 'a'), (9, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: insert into ice_stale_partlevel values (9, 'a'), (9, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: explain select id from ice_stale_partlevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_partlevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_partlevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_partlevel +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: drop table ice_stale_partlevel +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_partlevel diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out new file mode 100644 index 000000000000..4a35daa7f9af --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out @@ -0,0 +1,377 @@ +PREHOOK: query: create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_src +POSTHOOK: query: create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_src +PREHOOK: query: insert into ice_src values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_src +POSTHOOK: query: insert into ice_src values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_src +PREHOOK: query: create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@ice_src +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: Lineage: ice_ctas.id SIMPLE [(ice_src)ice_src.FieldSchema(name:id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_ctas.p SIMPLE [(ice_src)ice_src.FieldSchema(name:p, type:string, comment:null), ] +PREHOOK: query: explain select id from ice_ctas where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_ctas where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_ctas + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_ctas +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_ctas +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_ctas +POSTHOOK: query: drop table ice_ctas +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_ctas +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_ctas +PREHOOK: query: create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: insert into ice_unpart_w values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: insert into ice_unpart_w values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: analyze table ice_unpart_w compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_unpart_w compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_unpart_w where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_unpart_w where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart_w + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_unpart_w values (9, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: insert into ice_unpart_w values (9, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: explain select id from ice_unpart_w where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_unpart_w where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart_w + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_unpart_w +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: drop table ice_unpart_w +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_iow +POSTHOOK: query: create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_iow +PREHOOK: query: insert into ice_iow values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_iow +POSTHOOK: query: insert into ice_iow values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_iow +PREHOOK: query: analyze table ice_iow compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_iow +PREHOOK: Output: default@ice_iow +PREHOOK: Output: default@ice_iow@p=a +PREHOOK: Output: default@ice_iow@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_iow compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: default@ice_iow +POSTHOOK: Output: default@ice_iow@p=a +POSTHOOK: Output: default@ice_iow@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_iow where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_iow +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_iow where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_iow + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert overwrite table ice_iow select id + 100, p from ice_src +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_src +PREHOOK: Output: default@ice_iow +POSTHOOK: query: insert overwrite table ice_iow select id + 100, p from ice_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: default@ice_iow +PREHOOK: query: explain select id from ice_iow where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_iow +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_iow where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_iow + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_iow +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_iow +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_iow +POSTHOOK: query: drop table ice_iow +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_iow +PREHOOK: query: drop table ice_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_src +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_src +POSTHOOK: query: drop table ice_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_src diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out index 0032d06a3584..f6b8cfea653d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out @@ -65,7 +65,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### "type": "VARCHAR", "nullable": true, "precision": 2147483647, - "name": "PARTITION__PROJECTION" + "name": "PARTITION__NAME" }, { "type": "BIGINT", diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index e1e5d28cf61b..e17468137436 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -67,34 +67,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -150,7 +146,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -184,34 +180,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -267,7 +259,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -275,14 +267,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -291,38 +283,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -378,7 +366,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -386,14 +374,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -402,38 +390,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -572,7 +556,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -606,34 +590,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -690,7 +670,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -724,34 +704,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -808,7 +784,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -842,34 +818,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -941,7 +913,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -966,18 +938,15 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -999,17 +968,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1064,7 +1032,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1089,18 +1057,15 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1122,17 +1087,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1187,7 +1151,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1212,18 +1176,15 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.9583333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1245,17 +1206,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1310,7 +1270,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1335,18 +1295,15 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.9791667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1368,17 +1325,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1435,7 +1391,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1460,18 +1416,15 @@ STAGE PLANS: Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.9895833 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1493,17 +1446,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1560,7 +1512,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1585,18 +1537,15 @@ STAGE PLANS: Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1618,17 +1567,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2153,7 +2101,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2187,34 +2135,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2270,7 +2214,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2304,34 +2248,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2387,7 +2327,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2421,34 +2361,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2518,7 +2454,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2544,35 +2480,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2628,7 +2560,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2654,35 +2586,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2847,7 +2775,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2881,34 +2809,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2964,7 +2888,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2998,34 +2922,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3081,7 +3001,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3115,34 +3035,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3212,7 +3128,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3238,35 +3154,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3322,7 +3234,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3348,35 +3260,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3539,7 +3447,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3573,34 +3481,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3656,7 +3560,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3690,34 +3594,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3773,7 +3673,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3807,34 +3707,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3904,7 +3800,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3930,35 +3826,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4014,7 +3906,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4040,35 +3932,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4231,7 +4119,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4265,34 +4153,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4348,7 +4232,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4382,34 +4266,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4465,7 +4345,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4499,34 +4379,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4596,21 +4472,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -4619,38 +4495,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4706,21 +4578,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -4729,38 +4601,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index a0ad059f0a83..16bc406a9464 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -69,34 +69,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -152,7 +148,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -186,34 +182,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -269,7 +261,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -303,34 +295,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -400,21 +388,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -423,38 +411,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -510,21 +494,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -533,38 +517,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -729,7 +709,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -763,34 +743,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -846,7 +822,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -880,34 +856,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -963,7 +935,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -997,34 +969,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1094,21 +1062,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), DATE'1999-12-13' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1117,38 +1085,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1204,21 +1168,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), DATE'1999-12-02' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1227,38 +1191,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1423,7 +1383,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1457,34 +1417,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1540,7 +1496,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1574,34 +1530,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1657,7 +1609,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1691,34 +1643,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1788,21 +1736,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-13' (type: date), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1811,38 +1759,34 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1898,21 +1842,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-02' (type: date), 234675894076895090L (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1921,38 +1865,34 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2117,7 +2057,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2151,34 +2091,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2234,7 +2170,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2268,34 +2204,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2351,7 +2283,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2377,35 +2309,31 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.75 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2554,7 +2482,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2588,34 +2516,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2671,7 +2595,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2705,34 +2629,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2788,7 +2708,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2814,35 +2734,31 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.75 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2992,7 +2908,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3025,18 +2941,15 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(38,0)), _col2 (type: decimal(38,0)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(38,0)), _col1 (type: decimal(38,0)), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -3057,17 +2970,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DECIMAL' (type: string), _col1 (type: decimal(38,0)), _col2 (type: decimal(38,0)), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DECIMAL' (type: string), _col0 (type: decimal(38,0)), _col1 (type: decimal(38,0)), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 55905a907b95..7a7e3dc091fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -56,7 +56,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -90,34 +90,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), max(length(b)), avg(COALESCE(length(b),0)), count(b), compute_bit_vector_hll(b) - keys: iceberg_truncate(b, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: struct), _col8 (type: bigint), _col9 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), UDFToLong(_col1) (type: bigint), UDFToLong(_col2) (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint), COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), named_struct('b_trunc_2',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 754 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), UDFToLong(_col0) (type: bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint), COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 754 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 9212bb8dc3c8..bfd4698b73b3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -57,14 +57,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -73,38 +73,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -160,7 +156,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -168,14 +164,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -184,38 +180,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -277,7 +269,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"intcol\":\"true\",\"pcol\":\"true\",\"strcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"strcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"int\"}]} @@ -358,7 +350,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -392,34 +384,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -497,7 +485,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -522,18 +510,15 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.75 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -555,17 +540,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -620,7 +604,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -645,18 +629,15 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.75 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -678,17 +659,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -743,7 +723,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -768,18 +748,15 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.75 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -801,17 +778,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -866,43 +842,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -913,7 +886,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -924,17 +897,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -991,43 +963,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1038,7 +1007,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1049,17 +1018,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1116,43 +1084,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.95 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1163,7 +1128,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1174,17 +1139,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1248,7 +1212,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -1371,7 +1335,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1405,34 +1369,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1500,21 +1460,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-31' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1523,38 +1483,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1610,21 +1566,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-26' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1633,38 +1589,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1726,7 +1678,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -1837,7 +1789,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1871,34 +1823,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1966,21 +1914,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 34567890123456787L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1989,38 +1937,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2076,21 +2020,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2099,38 +2043,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2192,7 +2132,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"long\"}]} @@ -2303,7 +2243,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2337,34 +2277,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2432,21 +2368,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2455,38 +2391,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2542,21 +2474,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2565,38 +2497,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2658,7 +2586,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"double\"}]} @@ -2769,7 +2697,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2803,34 +2731,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2898,7 +2822,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2924,35 +2848,31 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3008,7 +2928,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3034,35 +2954,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3124,7 +3040,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(10, 6)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 547b1e0d3d68..5e29ac3cd1c4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -69,34 +69,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -152,7 +148,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -186,34 +182,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -269,7 +261,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -303,34 +295,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -396,21 +384,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -419,38 +407,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -506,21 +490,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -529,38 +513,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -703,7 +683,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -737,34 +717,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -820,7 +796,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -854,34 +830,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -937,7 +909,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -971,34 +943,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1064,21 +1032,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), DATE'1999-12-13' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1087,38 +1055,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1174,21 +1138,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), DATE'1999-12-02' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1197,38 +1161,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1371,7 +1331,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1405,34 +1365,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1488,7 +1444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1522,34 +1478,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1605,7 +1557,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1639,34 +1591,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1736,7 +1684,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1762,35 +1710,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1846,7 +1790,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1872,35 +1816,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1962,7 +1902,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2065,7 +2005,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2099,34 +2039,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2182,7 +2118,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2216,34 +2152,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2299,7 +2231,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2325,35 +2257,31 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out index bba70b438c06..f24d9836fbaf 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out @@ -24,7 +24,7 @@ POSTHOOK: Input: default@ice_parquet__temp_table_for_load_data__ POSTHOOK: Output: default@ice_parquet Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -40,7 +40,7 @@ Stage-3 Select Operator [SEL_19] Output:["_col0","_col1","_col2","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_15] + PARTITION_ONLY_SHUFFLE [RS_15] PartitionCols:_col2 Select Operator [SEL_14] (rows=77 width=187) Output:["_col0","_col1","_col2"] @@ -48,15 +48,14 @@ Stage-3 default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"] Reducer 3 vectorized File Output Operator [FS_23] - Select Operator [SEL_22] (rows=38 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_21] (rows=38 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=77 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol + Select Operator [SEL_22] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_21] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_18] + Group By Operator [GBY_17] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"] Select Operator [SEL_16] (rows=77 width=187) Output:["strcol","intcol","pcol"] Please refer to the previous Select Operator [SEL_14] @@ -79,7 +78,7 @@ POSTHOOK: Input: default@ice_parquet__temp_table_for_load_data__ POSTHOOK: Output: default@ice_parquet Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -95,7 +94,7 @@ Stage-3 Select Operator [SEL_19] Output:["_col0","_col1","_col2","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_15] + PARTITION_ONLY_SHUFFLE [RS_15] PartitionCols:_col2 Select Operator [SEL_14] (rows=77/6 width=187) Output:["_col0","_col1","_col2"] @@ -103,15 +102,14 @@ Stage-3 default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"] Reducer 3 vectorized File Output Operator [FS_23] - Select Operator [SEL_22] (rows=38/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_21] (rows=38/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=77/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol + Select Operator [SEL_22] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_21] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_18] + Group By Operator [GBY_17] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"] Select Operator [SEL_16] (rows=77/6 width=187) Output:["strcol","intcol","pcol"] Please refer to the previous Select Operator [SEL_14] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out index 4cd3f87b9aaf..4eac820ba403 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -202,7 +202,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -210,14 +210,14 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [6, 7, 8, 9, 10, 11, 1, 2, 3, 4, 5] + projectedOutputColumnNums: [6, 7, 8, 9, 11, 1, 2, 3, 4, 5] selectExpressions: ConstantVectorExpression(val 1090969) -> 11:bigint Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumns: 4:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 11:bigint, 1:string, 2:string, 3:int, 5:string + valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 11:bigint, 1:string, 2:string, 3:int, 5:string Filter Vectorization: className: VectorFilterOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out new file mode 100644 index 000000000000..35dc90b880d7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -0,0 +1,827 @@ +PREHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: analyze table ice_part_stats compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: default@ice_part_stats@p=b +PREHOOK: Output: default@ice_part_stats@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: default@ice_part_stats@p=b +POSTHOOK: Output: default@ice_part_stats@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_part_stats values (11, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (11, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p) IN ('a', 'b') (type: boolean) + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: drop table ice_part_stats +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: drop table ice_part_stats +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: insert into ice_unpart values (1), (5), (9) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (1), (5), (9) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: analyze table ice_unpart compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_unpart compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_unpart values (11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: insert into ice_unpart values (20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +20 +PREHOOK: query: drop table ice_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: drop table ice_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: analyze table ice_tbl_level compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level@p=a +PREHOOK: Output: default@ice_tbl_level@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_tbl_level compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level@p=a +POSTHOOK: Output: default@ice_tbl_level@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: explain +select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 +PREHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_tbl_level + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_tbl_level +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: drop table ice_tbl_level +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats_hms + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_part_stats_hms +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: drop table ice_part_stats_hms +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out index b87b1a62286b..8236bb2c9609 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out @@ -131,9 +131,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice_pp_key - Statistics: Num rows: 10 Data size: 63810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 63730 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 10 Data size: 63810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 63730 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.9 @@ -151,9 +151,9 @@ STAGE PLANS: TableScan alias: tbl_ice_pp_key As of version: s1 - Statistics: Num rows: 2 Data size: 12636 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 12620 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 2 Data size: 12636 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 12620 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.5 @@ -272,19 +272,19 @@ STAGE PLANS: TableScan alias: tbl_ice_pp_key filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 8 Data size: 51174 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 8 Data size: 51174 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - minReductionHashAggr: 0.875 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 4 @@ -293,22 +293,22 @@ STAGE PLANS: alias: tbl_ice_pp_key As of version: s1 filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Filter Operator predicate: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 2 @@ -318,14 +318,14 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'current' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -337,14 +337,14 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL Select Operator expressions: 'asof_s1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out index fae1fb8c26df..fd34901d33a2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -58,14 +58,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is null (type: boolean) - Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -109,14 +109,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out index c6365d049e94..c5146d8505e9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out @@ -53,15 +53,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice01 - filterExpr: ((year = 2023) and (month = 10) and (day = 3)) (type: boolean) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + filterExpr: ((day = 3) and (year = 2023) and (month = 10)) (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index ea652fe65e98..91f65d1827a1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -85,17 +85,17 @@ STAGE PLANS: TableScan alias: test_ice_int filterExpr: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -280,17 +280,17 @@ STAGE PLANS: TableScan alias: test_ice_bigint filterExpr: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 297 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 226784902765739L (type: bigint), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 226784902765739L (type: bigint), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -501,17 +501,17 @@ STAGE PLANS: TableScan alias: test_ice_str filterExpr: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 693 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), 'ddd' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), 'ddd' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -742,17 +742,17 @@ STAGE PLANS: TableScan alias: test_ice_date filterExpr: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -933,14 +933,14 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), b (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), b (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1111,18 +1111,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test_ice_double_date - filterExpr: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + filterExpr: ((b = DATE'2022-02-07') and (a = 1.156748927566759E11D)) (type: boolean) + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + predicate: ((b = DATE'2022-02-07') and (a = 1.156748927566759E11D)) (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out index ee4e941ebbd0..1b75e3b7acd7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out @@ -123,13 +123,13 @@ Stage-0 Filter Operator [FIL_14] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_2] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -194,13 +194,13 @@ Stage-0 Filter Operator [FIL_18] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_2] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -257,13 +257,13 @@ Stage-0 Filter Operator [FIL_14] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_2] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -328,13 +328,13 @@ Stage-0 Filter Operator [FIL_18] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_2] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -391,13 +391,13 @@ Stage-0 Filter Operator [FIL_27] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_31] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_30] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -462,13 +462,13 @@ Stage-0 Filter Operator [FIL_41] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_48] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_47] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -525,13 +525,13 @@ Stage-0 Filter Operator [FIL_27] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_31] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_30] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -596,13 +596,13 @@ Stage-0 Filter Operator [FIL_41] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_48] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_47] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out index 7ce866f2b03c..99971bfe6268 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out @@ -271,13 +271,13 @@ Stage-3 Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] <-Select Operator [SEL_29] (rows=1000 width=95) Output:["_col0","_col1"] Filter Operator [FIL_28] (rows=1000 width=95) predicate:key is not null TableScan [TS_3] (rows=1000 width=95) - default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] + default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] @@ -435,13 +435,13 @@ Stage-3 Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] <-Select Operator [SEL_29] (rows=524 width=95) Output:["_col0","_col1"] Filter Operator [FIL_28] (rows=524 width=95) predicate:key is not null TableScan [TS_3] (rows=524 width=95) - default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out index f21a3c84da24..bb6dc3571e76 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out @@ -139,13 +139,13 @@ Stage-0 Filter Operator [FIL_21] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] Filter Operator [FIL_24] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b @@ -213,13 +213,13 @@ Stage-0 Filter Operator [FIL_26] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] Filter Operator [FIL_29] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out index dd93fcf31851..158c2ffc10b2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out @@ -252,13 +252,13 @@ Stage-0 Filter Operator [FIL_21] (rows=738 width=89) predicate:(part is not null and key is not null) TableScan [TS_3] (rows=738 width=89) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] <-Select Operator [SEL_25] (rows=738 width=4) Output:["_col0"] Filter Operator [FIL_24] (rows=738 width=89) predicate:(part is not null and key is not null) TableScan [TS_0] (rows=738 width=89) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b @@ -319,7 +319,7 @@ Stage-0 Filter Operator [FIL_27] (rows=738 width=89) predicate:(part is not null and key is not null) TableScan [TS_3] (rows=738 width=89) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=85) Group By Operator [GBY_31] (rows=2 width=85) Output:["_col0"],keys:_col0 @@ -331,7 +331,7 @@ Stage-0 Filter Operator [FIL_33] (rows=738 width=89) predicate:(part is not null and key is not null) TableScan [TS_0] (rows=738 width=89) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out index 8ff5e300aa46..45295e39b953 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out @@ -191,13 +191,13 @@ Stage-0 Filter Operator [FIL_26] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] Filter Operator [FIL_29] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b @@ -258,13 +258,13 @@ Stage-0 Filter Operator [FIL_21] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] Filter Operator [FIL_24] (rows=238 width=89) predicate:((part = '1') and key is not null) TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out index 024fde7f1576..153fdc311b89 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out @@ -161,13 +161,13 @@ Stage-3 Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] <-Select Operator [SEL_29] (rows=524 width=95) Output:["_col0","_col1"] Filter Operator [FIL_28] (rows=524 width=95) predicate:key is not null TableScan [TS_3] (rows=524 width=95) - default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out index 7b486db67697..520b925a0bf7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out @@ -74,7 +74,7 @@ Stage-0 Filter Operator [FIL_39] (rows=500 width=269) predicate:(key1 is not null and key2 is not null) TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -172,7 +172,7 @@ Stage-0 Filter Operator [FIL_39] (rows=500 width=269) predicate:((key1 <> '0') and (key1 <> '100') and key2 is not null) TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -270,7 +270,7 @@ Stage-0 Filter Operator [FIL_29] (rows=500 width=269) predicate:key1 is not null TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -368,7 +368,7 @@ Stage-0 Filter Operator [FIL_29] (rows=500 width=269) predicate:((key1 <> '0') and (key1 <> '100')) TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -468,7 +468,7 @@ Stage-0 Filter Operator [FIL_29] (rows=500 width=269) predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null) TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -566,7 +566,7 @@ Stage-0 Filter Operator [FIL_29] (rows=500 width=269) predicate:key2 is not null TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -662,7 +662,7 @@ Stage-0 Filter Operator [FIL_44] (rows=500 width=269) predicate:(key1 is not null and key2 is not null and value is not null) TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out index 148f2d89ee20..0afae5cee90d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out @@ -148,11 +148,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=16 width=8) + Select Operator [SEL_31] (rows=11 width=7) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=16 width=8) + Map Join Operator [MAPJOIN_29] (rows=11 width=7) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -163,12 +163,12 @@ Stage-0 predicate:key1 is not null TableScan [TS_3] (rows=6 width=3) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1"] - <-Select Operator [SEL_28] (rows=16 width=8) + <-Select Operator [SEL_28] (rows=11 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=8) + Filter Operator [FIL_27] (rows=11 width=7) predicate:key1 is not null - TableScan [TS_0] (rows=16 width=8) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] + TableScan [TS_0] (rows=16 width=7) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] PREHOOK: query: SELECT a.key1, a.id FROM srcbucket_big a @@ -232,11 +232,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=16 width=92) + Select Operator [SEL_31] (rows=11 width=75) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=16 width=92) + Map Join Operator [MAPJOIN_29] (rows=11 width=75) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -247,12 +247,12 @@ Stage-0 predicate:key2 is not null TableScan [TS_3] (rows=6 width=72) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key2"] - <-Select Operator [SEL_28] (rows=16 width=92) + <-Select Operator [SEL_28] (rows=11 width=75) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=92) + Filter Operator [FIL_27] (rows=11 width=75) predicate:key2 is not null - TableScan [TS_0] (rows=16 width=92) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] + TableScan [TS_0] (rows=16 width=69) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] PREHOOK: query: SELECT a.key2, a.id FROM srcbucket_big a @@ -316,11 +316,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_37] - Select Operator [SEL_36] (rows=16 width=77) + Select Operator [SEL_36] (rows=16 width=76) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_35] - Map Join Operator [MAPJOIN_34] (rows=16 width=77) + Map Join Operator [MAPJOIN_34] (rows=16 width=76) Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1"] <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_31] @@ -331,11 +331,11 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=6 width=74) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Select Operator [SEL_33] (rows=16 width=77) + <-Select Operator [SEL_33] (rows=16 width=76) Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=16 width=77) + Filter Operator [FIL_32] (rows=16 width=76) predicate:value is not null - TableScan [TS_0] (rows=21 width=77) + TableScan [TS_0] (rows=21 width=76) default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["value","id"] PREHOOK: query: SELECT a.value, a.id @@ -400,11 +400,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_42] - Select Operator [SEL_41] (rows=12 width=96) + Select Operator [SEL_41] (rows=3 width=64) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_40] - Map Join Operator [MAPJOIN_39] (rows=12 width=96) + Map Join Operator [MAPJOIN_39] (rows=3 width=64) BucketMapJoin:true,Conds:SEL_38._col0, _col1=RS_36._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_36] @@ -415,12 +415,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null) TableScan [TS_3] (rows=6 width=75) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2"] - <-Select Operator [SEL_38] (rows=12 width=96) + <-Select Operator [SEL_38] (rows=4 width=72) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=12 width=96) + Filter Operator [FIL_37] (rows=4 width=72) predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=96) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] + TableScan [TS_0] (rows=12 width=64) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] PREHOOK: query: SELECT a.key1, a.key2, a.id FROM srcbucket_big a @@ -480,11 +480,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_47] - Select Operator [SEL_46] (rows=9 width=78) + Select Operator [SEL_46] (rows=6 width=66) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_45] - Map Join Operator [MAPJOIN_44] (rows=9 width=78) + Map Join Operator [MAPJOIN_44] (rows=6 width=66) BucketMapJoin:true,Conds:SEL_43._col0, _col1=RS_41._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_41] @@ -495,12 +495,12 @@ Stage-0 predicate:(key1 is not null and value is not null) TableScan [TS_3] (rows=6 width=77) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","value"] - <-Select Operator [SEL_43] (rows=12 width=83) + <-Select Operator [SEL_43] (rows=8 width=73) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=12 width=83) - predicate:(value is not null and key1 is not null) - TableScan [TS_0] (rows=16 width=81) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] + Filter Operator [FIL_42] (rows=8 width=73) + predicate:(key1 is not null and value is not null) + TableScan [TS_0] (rows=16 width=73) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] PREHOOK: query: SELECT a.key1, a.value, a.id FROM srcbucket_big a @@ -560,11 +560,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_52] - Select Operator [SEL_51] (rows=5 width=150) + Select Operator [SEL_51] (rows=2 width=184) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_50] - Map Join Operator [MAPJOIN_49] (rows=5 width=150) + Map Join Operator [MAPJOIN_49] (rows=2 width=184) BucketMapJoin:true,Conds:SEL_48._col0, _col1, _col2=RS_46._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_46] @@ -575,12 +575,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null and value is not null) TableScan [TS_3] (rows=6 width=150) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] - <-Select Operator [SEL_48] (rows=8 width=164) + <-Select Operator [SEL_48] (rows=2 width=184) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=8 width=164) - predicate:(value is not null and key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=164) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] + Filter Operator [FIL_47] (rows=2 width=184) + predicate:(key1 is not null and key2 is not null and value is not null) + TableScan [TS_0] (rows=12 width=124) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] PREHOOK: query: SELECT a.key1, a.key2, a.value, a.id FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out new file mode 100644 index 000000000000..af2e3e249bd8 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out @@ -0,0 +1,342 @@ +PREHOOK: query: create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (1, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (1, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (2, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (2, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (3, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (3, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: explain select id from ice_comp_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: alter table ice_comp_unpart COMPACT 'major' and wait +PREHOOK: type: ALTERTABLE_COMPACT +PREHOOK: Input: default@ice_comp_unpart +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: alter table ice_comp_unpart COMPACT 'major' and wait +POSTHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: Input: default@ice_comp_unpart +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: explain select id from ice_comp_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_comp_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: select min(id), max(id) from ice_comp_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +1 7 +PREHOOK: query: drop table ice_comp_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_comp_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: drop table ice_comp_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_comp_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp +POSTHOOK: query: create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (1, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (1, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (2, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (2, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (3, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (3, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (4, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (4, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: explain select id from ice_comp where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait +PREHOOK: type: ALTERTABLE_COMPACT +PREHOOK: Input: default@ice_comp +PREHOOK: Output: default@ice_comp@p=a +POSTHOOK: query: alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait +POSTHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: Input: default@ice_comp +POSTHOOK: Output: default@ice_comp@p=a +PREHOOK: query: explain select id from ice_comp where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_comp +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: select min(id), max(id) from ice_comp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +1 7 +PREHOOK: query: drop table ice_comp +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_comp +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp +POSTHOOK: query: drop table ice_comp +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_comp +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out index b4b2ff455e58..30d0b3d9ee3a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out @@ -704,7 +704,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -737,18 +737,15 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ts), max(ts), count(1), count(ts), compute_bit_vector_hll(ts), min(dd), max(dd), count(dd), compute_bit_vector_hll(dd), min(ll), max(ll), count(ll), compute_bit_vector_hll(ll) - keys: iceberg_bucket(ll, 4) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -771,17 +768,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'TIMESTAMP' (type: string), _col1 (type: timestamp), _col2 (type: timestamp), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DOUBLE' (type: string), _col6 (type: double), _col7 (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('ll_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 907 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'TIMESTAMP' (type: string), _col0 (type: timestamp), _col1 (type: timestamp), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DOUBLE' (type: string), _col5 (type: double), _col6 (type: double), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 863 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 907 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 863 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8e77ed6fd0a9..50bd74b91a9a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -598,17 +598,17 @@ STAGE PLANS: alias: ice_orc.tag_v1 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v1 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -926,7 +926,7 @@ POSTHOOK: type: SHOW COMPACTIONS CompactionId Database Table Partition Type State Worker host Worker Enqueue Time Start Time Duration(ms) HadoopJobId Error message Initiator host Initiator Pool name TxnId Next TxnId Commit Time Highest WriteId #Masked# default ice_orc company_id=100/dept_id=1 MAJOR succeeded #Masked# manual iceberg 0 0 0 --- #Masked# default ice_orc company_id=100/dept_id=2 MAJOR succeeded #Masked# manual iceberg 0 0 0 --- -#Masked# default ice_orc company_id=null/dept_id=null MAJOR refused #Masked# manual iceberg 0 0 0 --- +#Masked# default ice_orc company_id=__HIVE_DEFAULT_PARTITION__/dept_id=__HIVE_DEFAULT_PARTITION__ MAJOR refused #Masked# manual iceberg 0 0 0 --- #Masked# default ice_orc --- MAJOR succeeded #Masked# manual iceberg 0 0 0 --- PREHOOK: query: select `partition`, spec_id, content, record_count from default.ice_orc.files @@ -965,14 +965,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1008,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index 60ad0af240d3..3e8bda76a610 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -154,7 +154,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -239,7 +239,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -337,7 +337,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 3bb172c0df60..987c779d87fb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -171,7 +171,7 @@ CompactionId Database Table Partition Type State Worker host Worker Enqueue Time #Masked# default srcbucket_big key_bucket_8=0 MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big key_bucket_8=3 MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big key_bucket_8=4 MINOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default srcbucket_big key_bucket_8=null MINOR succeeded #Masked# manual default 0 0 0 --- +#Masked# default srcbucket_big key_bucket_8=__HIVE_DEFAULT_PARTITION__ MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big --- MINOR succeeded #Masked# manual default 0 0 0 --- PREHOOK: query: desc formatted default.srcbucket_big PREHOOK: type: DESCTABLE diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 466c234e72c5..0de0774396b6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -403,7 +403,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 4 @@ -490,7 +490,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out index 79536df6215d..a1eace49222e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out @@ -266,7 +266,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index 37bff7329394..51f0da923f1f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -235,7 +235,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -390,7 +390,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -916,7 +916,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index c215c4005a46..e2669a45789d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -139,7 +139,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -359,7 +359,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -520,7 +520,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -655,7 +655,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -768,7 +768,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -942,7 +942,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index b2ea3e67b861..0b66a1287bca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -558,7 +558,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index b6e253358ee6..07ba4289b8f6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -135,52 +135,52 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 100) (type: boolean) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Execution mode: vectorized Map 12 Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL @@ -200,9 +200,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -221,21 +221,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -243,30 +243,30 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 972 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -278,24 +278,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or _col5 is null or _col8 is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col4 <> _col7) or _col4 is null or _col7 is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -304,15 +304,15 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -382,16 +382,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -472,16 +472,16 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -506,30 +506,30 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: _col5 is null (type: boolean) + predicate: _col4 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), _col6 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out index 115992d60112..bb4c64a103ba 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out @@ -148,41 +148,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a <= 100) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -201,24 +201,24 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Execution mode: vectorized Reducer 10 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -237,17 +237,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -259,20 +259,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 2440 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 14 Data size: 1520 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1952 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1216 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new') (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3424 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new') (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2688 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -285,16 +285,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 486 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -306,24 +306,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 is null or (_col5 = _col8)) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 is null or (_col4 = _col7)) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -332,11 +332,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -411,16 +411,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -483,44 +483,44 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a > 100) (type: boolean) Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: ((a > 100) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE @@ -591,20 +591,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 14 Data size: 2420 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 14 Data size: 1500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new 2') (type: string), null (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3412 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new 2') (type: string), null (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -616,24 +616,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 <= 100) or (_col4 is null or ((_col4 = _col7) and (_col4 > 100))) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -642,11 +642,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -659,23 +659,23 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -694,17 +694,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -865,41 +865,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) + Map-reduce partition columns: _col3 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -959,16 +959,16 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col5 is null or (_col5 <> _col8) or _col8 is null) (type: boolean) + predicate: (_col4 is null or (_col4 <> _col7) or _col7 is null) (type: boolean) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -976,7 +976,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -985,7 +985,7 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1001,9 +1001,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col4 (type: int) + 0 _col3 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -1011,13 +1011,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1042,8 +1042,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1059,13 +1059,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), 'Merged' (type: string), (_col6 - 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), 'Merged' (type: string), (_col5 - 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out index dcb3ecd97145..830129a81e73 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out @@ -99,16 +99,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -117,48 +117,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -170,10 +170,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index 693513fa2dfc..c2529066adac 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -101,16 +101,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -119,48 +119,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -171,10 +171,10 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: PARTIAL + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out index 64bbd1b55dec..ca720205bf24 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out @@ -71,31 +71,31 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[+($6, 100)], t__c=[$7]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[+($5, 100)], t__c=[$6]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), IS NULL($5)), >($5, 10), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), IS NOT NULL($5)), <=($5, 10), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), IS NULL($4)), >($4, 10), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), IS NOT NULL($4)), <=($4, 10), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -111,11 +111,11 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[>($0, 10)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -141,23 +141,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(IS NULL($5), AND(=($5, $8), =($6, $9), >($5, 10)), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), IS NOT NULL($5)))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), IS NOT NULL($4)))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -174,12 +174,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(IS NULL($3), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveTableScan(table=[[default, source]], table:alias=[s]) @@ -203,21 +203,21 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[null:INTEGER], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6), IS NULL($7))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[null:INTEGER], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5), IS NULL($6))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -234,12 +234,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) @@ -264,23 +264,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -297,12 +297,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) diff --git a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out index bfa7597c38e0..d37561e23c54 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out @@ -373,12 +373,12 @@ STAGE PLANS: predicate: (b < 5) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out index f6adb53ea76c..5fbd6803a7c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out @@ -410,16 +410,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string) + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -474,25 +474,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1524 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint) + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -520,9 +520,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Filter Operator predicate: _col3 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -552,12 +552,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out index 7fbe53177787..dabdc0989f39 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out @@ -153,16 +153,16 @@ STAGE PLANS: alias: default.mat2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), _c4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), _c4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col10 (type: double) + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: double) Execution mode: vectorized Map 6 Map Operator Tree: @@ -217,25 +217,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 2172 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 6 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col10 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col9 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: double) + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -263,9 +263,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1448 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -295,12 +295,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint), VALUE._col8 (type: bigint), VALUE._col9 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint), VALUE._col7 (type: bigint), VALUE._col8 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out index 94381e415353..a52e9ae8390f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out @@ -120,8 +120,8 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -129,7 +129,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -165,25 +165,25 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) nullSafes: [true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: bigint) Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -198,7 +198,7 @@ STAGE PLANS: predicate: _col2 is null (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -213,8 +213,8 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out index 22ccaad5e4da..23948e8892ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out @@ -87,16 +87,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE + expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) null sort order: zzz sort order: +++ Map-reduce partition columns: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -135,25 +135,25 @@ STAGE PLANS: 0 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) 1 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) nullSafes: [true, true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: varchar(256)), _col6 (type: char(100)), _col7 (type: int), _col8 (type: bigint) + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: varchar(256)), _col5 (type: char(100)), _col6 (type: int), _col7 (type: bigint) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -181,9 +181,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -213,12 +213,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: varchar(256)), VALUE._col6 (type: char(100)), VALUE._col7 (type: int), VALUE._col8 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: varchar(256)), VALUE._col5 (type: char(100)), VALUE._col6 (type: int), VALUE._col7 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 0ff1c86f2107..7b1f159b780d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -266,7 +266,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -282,7 +282,7 @@ Stage-3 Select Operator [SEL_25] Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col4","_col5"] <-Reducer 2 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_21] + PARTITION_ONLY_SHUFFLE [RS_21] PartitionCols:_col4, _col5 Select Operator [SEL_20] (rows=11 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -298,15 +298,14 @@ Stage-3 default@llap_orders,llap_orders,Tbl:COMPLETE,Col:NONE,Output:["orderid","quantity","itemid","tradets","p1","p2"] Reducer 4 vectorized File Output Operator [FS_29] - Select Operator [SEL_28] (rows=5 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36"] - Group By Operator [GBY_27] (rows=5 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)","min(VALUE._col13)","max(VALUE._col14)","count(VALUE._col15)","compute_bit_vector_hll(VALUE._col16)","max(VALUE._col17)","avg(VALUE._col18)","count(VALUE._col19)","compute_bit_vector_hll(VALUE._col20)","max(VALUE._col21)","avg(VALUE._col22)","count(VALUE._col23)","compute_bit_vector_hll(VALUE._col24)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_24] - PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=11 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"],keys:p1, p2 + Select Operator [SEL_28] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] + Group By Operator [GBY_27] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)","min(VALUE._col13)","max(VALUE._col14)","count(VALUE._col15)","compute_bit_vector_hll(VALUE._col16)","max(VALUE._col17)","avg(VALUE._col18)","count(VALUE._col19)","compute_bit_vector_hll(VALUE._col20)","max(VALUE._col21)","avg(VALUE._col22)","count(VALUE._col23)","compute_bit_vector_hll(VALUE._col24)"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_24] + Group By Operator [GBY_23] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"] Select Operator [SEL_22] (rows=11 width=420) Output:["orderid","quantity","itemid","tradets","p1","p2"] Please refer to the previous Select Operator [SEL_20] diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 7dac902dc9ae..0db0f72d011d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -54,18 +54,18 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1461 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Map 4 Map Operator Tree: @@ -77,16 +77,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -96,34 +96,34 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -137,26 +137,26 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -175,28 +175,28 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col5 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -216,7 +216,7 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL @@ -347,131 +347,131 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: a (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Map 26 Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(c) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Reducer 10 @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,15 +497,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -514,22 +514,22 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -538,13 +538,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -553,19 +553,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -574,13 +574,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -589,26 +589,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 2447 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 2447 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -617,13 +617,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 1989 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 1989 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -632,15 +632,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2106 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2106 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -649,18 +649,18 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -668,46 +668,46 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -726,21 +726,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1029 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1029 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 21 Execution mode: vectorized Reduce Operator Tree: @@ -748,21 +748,21 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 22 Reduce Operator Tree: @@ -773,13 +773,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 23 Reduce Operator Tree: @@ -789,15 +789,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1809 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 1809 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 24 Reduce Operator Tree: Merge Join Operator @@ -806,15 +806,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 25 Execution mode: vectorized Reduce Operator Tree: @@ -822,11 +822,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 27 Execution mode: vectorized @@ -835,21 +835,21 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 28 Execution mode: vectorized @@ -858,38 +858,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 29 Execution mode: vectorized @@ -898,11 +898,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -913,11 +913,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -928,13 +928,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -945,23 +945,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -981,26 +981,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1009,29 +1009,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1794 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1794 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1177,8 +1177,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1186,55 +1186,55 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1244,11 +1244,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1259,20 +1259,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1292,26 +1292,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1320,22 +1320,22 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1354,21 +1354,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1552 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1584 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1378,60 +1378,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1440,29 +1440,29 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2799 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1475,17 +1475,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -1495,13 +1495,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -1510,18 +1510,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1529,7 +1529,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1538,19 +1538,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 @@ -1688,8 +1688,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1697,55 +1697,55 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1755,11 +1755,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1770,20 +1770,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1803,26 +1803,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1831,22 +1831,22 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1865,21 +1865,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1552 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1584 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1889,60 +1889,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1951,29 +1951,29 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1986,17 +1986,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -2006,13 +2006,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2021,18 +2021,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -2040,7 +2040,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -2049,19 +2049,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out index 150fa60ce166..9fb8158d7879 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out @@ -55,12 +55,12 @@ STAGE PLANS: predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1948 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -96,24 +96,24 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -132,17 +132,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -202,15 +202,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -327,151 +327,151 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) - minReductionHashAggr: 0.7777778 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: a (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 10 @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,26 +497,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -525,15 +525,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -548,13 +548,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 14 Reduce Operator Tree: @@ -565,11 +565,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 13 Data size: 2568 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 13 Data size: 2568 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -580,13 +580,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 16 Reduce Operator Tree: @@ -597,23 +597,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 13 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -633,26 +633,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -661,15 +661,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -678,13 +678,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -693,15 +693,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 20 Reduce Operator Tree: Merge Join Operator @@ -710,15 +710,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 21 Reduce Operator Tree: Merge Join Operator @@ -727,18 +727,18 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4437 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -751,16 +751,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 23 Execution mode: vectorized @@ -769,38 +769,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 24 Execution mode: vectorized @@ -809,11 +809,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 25 Execution mode: vectorized @@ -822,11 +822,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 26 Execution mode: vectorized @@ -835,38 +835,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 27 Execution mode: vectorized @@ -875,16 +875,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 28 Execution mode: vectorized @@ -893,11 +893,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 29 Execution mode: vectorized @@ -906,11 +906,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -920,13 +920,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -935,15 +935,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -952,25 +952,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -989,17 +989,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1013,13 +1013,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1028,19 +1028,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Union 7 Vertex: Union 7 @@ -1160,8 +1160,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1169,13 +1169,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1183,10 +1183,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1194,7 +1194,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1342,13 +1342,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1357,18 +1357,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1376,7 +1376,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1385,11 +1385,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1407,11 +1407,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1429,7 +1429,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1437,13 +1437,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1468,8 +1468,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1666,8 +1666,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1675,13 +1675,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1689,10 +1689,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1700,7 +1700,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1796,7 +1796,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1804,13 +1804,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1835,8 +1835,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1917,11 +1917,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1958,13 +1958,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1973,18 +1973,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1992,7 +1992,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2001,11 +2001,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out index ccf73acb8669..a33dcfab92b3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 6965a3604c45..47dde0580b33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -335,10 +335,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Select Operator expressions: ss_sold_time_sk (type: int), ss_item_sk2 (type: int), ss_customer_sk2 (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)), (floor((UDFToDouble(ss_item_sk2) / 1000.0D)) * 1000L) BETWEEN 1000L AND 2000L (type: boolean), (ss_ext_discount_amt < 0) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 @@ -347,7 +347,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -355,47 +355,47 @@ STAGE PLANS: 0 {_col22} {_col23} 1 keys: - 0 _col2 (type: int), _col1 (type: int) - 1 _col8 (type: int), _col7 (type: int) + 0 _col1 (type: int), _col2 (type: int) + 1 _col6 (type: int), _col7 (type: int) Map Join Vectorization: bigTableFilterExpressions: SelectColumnIsTrue(col 32:boolean), SelectColumnIsTrue(col 33:boolean) - bigTableKeyColumns: 3:int, 2:int + bigTableKeyColumns: 2:int, 3:int bigTableRetainColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] bigTableValueColumns: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) className: VectorMapJoinOuterMultiKeyOperator native: true nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - outerSmallTableKeyMapping: 2 -> 41, 3 -> 42 - projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) - smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + outerSmallTableKeyMapping: 2 -> 40, 3 -> 41 + projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) + smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) hashTableImplementationType: OPTIMIZED - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50 input vertices: 1 Map 5 Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col40 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col33 (type: int), _col39 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col38 (type: int), _col28 (type: string), _col6 (type: int), _col50 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col34 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col45 (type: decimal(7,2)), _col47 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col35 (type: int), _col31 (type: int), _col41 (type: decimal(7,2)), _col36 (type: int), _col4 (type: int), _col48 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col29 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col30 (type: int), _col42 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col51 (type: decimal(7,2)), _col32 (type: int), _col37 (type: int), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49 + expressions: _col39 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col32 (type: int), _col38 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col37 (type: int), _col6 (type: int), _col49 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col33 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col46 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col34 (type: int), _col30 (type: int), _col40 (type: decimal(7,2)), _col35 (type: int), _col4 (type: int), _col47 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col42 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col28 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col29 (type: int), _col41 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col50 (type: decimal(7,2)), _col31 (type: int), _col36 (type: int), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [50, 20, 43, 49, 34, 22, 36, 8, 4, 11, 59, 48, 38, 7, 60, 12, 44, 18, 15, 55, 57, 21, 9, 45, 41, 51, 46, 5, 58, 6, 14, 13, 54, 53, 2, 17, 39, 3, 16, 19, 35, 10, 40, 52, 1, 37, 61, 42, 47, 56] + projectedOutputColumnNums: [49, 20, 42, 48, 34, 22, 36, 8, 4, 11, 58, 47, 7, 59, 12, 43, 18, 15, 54, 56, 21, 9, 44, 40, 50, 45, 5, 57, 6, 14, 13, 53, 52, 2, 17, 38, 3, 16, 19, 35, 10, 39, 51, 1, 37, 60, 41, 46, 55] Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint), _col12 (type: string), 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), null (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint), 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), null (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [34, 35, 36, 37, 38, 30, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 62:decimal(7,2) + projectedOutputColumnNums: [34, 35, 36, 37, 30, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 61, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 61:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -412,17 +412,17 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [31, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 63, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 63:decimal(7,2) + projectedOutputColumnNums: [31, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 62, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 62:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -431,28 +431,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 42:int, 65:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int + keyColumns: 41:int, 64:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 31:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 63:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + valueColumns: 31:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 62:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 39:int)) - predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 40:int), SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 38:int)) + predicate: (_col23 is null and _col46 is null and _col35 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col44 (type: int), _col34 (type: int), _col37 (type: int), _col8 (type: int), _col27 (type: int), _col29 (type: int), _col13 (type: int), _col7 (type: int), _col22 (type: int), _col41 (type: int), _col9 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col35 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col39 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col5 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col43 (type: int), _col33 (type: int), _col36 (type: int), _col8 (type: int), _col26 (type: int), _col28 (type: int), _col12 (type: int), _col7 (type: int), _col21 (type: int), _col40 (type: int), _col9 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col29 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col37 (type: decimal(7,2)), _col34 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 64:int + projectedOutputColumnNums: [63, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 63:int Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -461,23 +461,23 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int + keyColumns: 3:int, 65:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 64:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 63:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col6 (type: string), _col40 (type: bigint), _col45 (type: bigint) - outputColumnNames: _col4, _col6, _col40, _col45 + expressions: _col4 (type: int), _col6 (type: string), _col39 (type: bigint), _col44 (type: bigint) + outputColumnNames: _col4, _col6, _col39, _col44 Select Vectorization: className: VectorSelectOperator native: true @@ -493,7 +493,7 @@ STAGE PLANS: native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] - keys: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint) + keys: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -526,7 +526,7 @@ STAGE PLANS: includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] Map 5 Map Operator Tree: TableScan @@ -535,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Filter Operator Filter Vectorization: className: VectorFilterOperator @@ -544,28 +544,28 @@ STAGE PLANS: predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [23, 24, 25, 26, 27, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + projectedOutputColumnNums: [23, 24, 25, 26, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col8 (type: int), _col7 (type: int) + key expressions: _col6 (type: int), _col7 (type: int) null sort order: zz sort order: ++ - Map-reduce partition columns: _col7 (type: int) + Map-reduce partition columns: _col6 (type: int) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: 3:int, 2:int + keyColumns: 2:int, 3:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int - valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: enabled: true @@ -909,16 +909,16 @@ Stage-6 Select Operator [SEL_50] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_46] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) Select Operator [SEL_44] (rows=5 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48"] Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) - BucketMapJoin:true,Conds:SEL_42._col2, _col1=RS_41._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51"] + BucketMapJoin:true,Conds:SEL_42._col1, _col2=RS_41._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] <-Map 5 [CUSTOM_EDGE] vectorized MULTICAST [RS_41] - PartitionCols:_col7 + PartitionCols:_col6 Select Operator [SEL_40] (rows=2 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"] Filter Operator [FIL_39] (rows=2 width=#Masked#) predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) TableScan [TS_2] (rows=2 width=#Masked#) @@ -926,7 +926,7 @@ Stage-6 <-Select Operator [SEL_42] (rows=5 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] TableScan [TS_0] (rows=5 width=#Masked#) - default@ssv,s,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + default@ssv,s,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] Reducer 3 vectorized File Output Operator [FS_61] table:{"name:":"default.store_sales"} @@ -938,7 +938,7 @@ Stage-6 Select Operator [SEL_51] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_47] (rows=1 width=#Masked#) - predicate:(_col24 is null and _col47 is null and _col36 is null) + predicate:(_col23 is null and _col46 is null and _col35 is null) Please refer to the previous Select Operator [SEL_44] Reducer 4 vectorized File Output Operator [FS_65] @@ -953,11 +953,11 @@ Stage-6 SHUFFLE [RS_57] PartitionCols:_col0, _col1, _col2, _col3 Group By Operator [GBY_56] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col40, _col6, _col45 + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col39, _col6, _col44 Select Operator [SEL_52] (rows=1 width=#Masked#) - Output:["_col4","_col6","_col40","_col45"] + Output:["_col4","_col6","_col39","_col44"] Filter Operator [FIL_48] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) Please refer to the previous Select Operator [SEL_44] Stage-7 Stats Work{} diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index eeb9cbfe354f..f94d7275f651 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -197,7 +197,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -351,7 +351,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -797,7 +797,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 0b7260d84206..734ac8f8f824 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -102,7 +102,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -282,7 +282,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -442,7 +442,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -576,7 +576,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -688,7 +688,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -861,7 +861,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index b4a16f6690d8..c2a1c1c06d12 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -480,7 +480,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out index 37f0589a9167..b052deb10003 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out @@ -234,17 +234,17 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: (a = 22) (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 485 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -321,12 +321,12 @@ STAGE PLANS: predicate: (c = 66) (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col9 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col8 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), 66 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), 66 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -337,7 +337,7 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Select Operator - expressions: 33 (type: int), _col9 (type: string), 66 (type: int) + expressions: 33 (type: int), _col8 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -456,16 +456,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -474,19 +474,19 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -497,11 +497,11 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -512,10 +512,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -527,10 +527,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: _col10 is null (type: boolean) + predicate: _col9 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -542,15 +542,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: (_col10 = _col1) (type: boolean) + predicate: (_col9 = _col1) (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -795,14 +795,14 @@ STAGE PLANS: TableScan alias: ice01.branch_test1 Snapshot ref: branch_test1 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java index 5e22dde0d97a..c73a90c26805 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java @@ -71,7 +71,11 @@ public void testIcebergCompactorWithAllPartitionFieldTypes() throws Exception{ CompactorTestUtil.runCompaction(conf, DB_NAME, TABLE_NAME, CompactionType.MINOR, false, "a=aaa111/a_trunc=aaa/a_bucket=0/b=1/c=100/d=1.0/e=2.0/f=4.00/g=true/h=2024-05-01/h_year=2024/i_month=2024-05/j_day=2024-05-01/k=2024-05-02T10%3A00%3A00/k_hour=2024-05-02-10", "a=bbb222/a_trunc=bbb/a_bucket=3/b=2/c=200/d=2.0/e=3.0/f=8.00/g=false/h=2024-05-03/h_year=2024/i_month=2024-05/j_day=2024-05-03/k=2024-05-04T13%3A00%3A00/k_hour=2024-05-04-13", - "a=null/a_trunc=null/a_bucket=null/b=null/c=null/d=null/e=null/f=null/g=null/h=null/h_year=null/i_month=null/j_day=null/k=null/k_hour=null" + "a=__HIVE_DEFAULT_PARTITION__/a_trunc=__HIVE_DEFAULT_PARTITION__/a_bucket=__HIVE_DEFAULT_PARTITION__/" + + "b=__HIVE_DEFAULT_PARTITION__/c=__HIVE_DEFAULT_PARTITION__/d=__HIVE_DEFAULT_PARTITION__/" + + "e=__HIVE_DEFAULT_PARTITION__/f=__HIVE_DEFAULT_PARTITION__/g=__HIVE_DEFAULT_PARTITION__/" + + "h=__HIVE_DEFAULT_PARTITION__/h_year=__HIVE_DEFAULT_PARTITION__/i_month=__HIVE_DEFAULT_PARTITION__/" + + "j_day=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__/k_hour=__HIVE_DEFAULT_PARTITION__" ); Assert.assertEquals(3, getFilesCount()); @@ -118,7 +122,7 @@ public void testIcebergAutoCompactionPartitionEvolution() throws Exception { // Compaction should be initiated for each partition from the latest spec Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=aaa", CompactionType.MINOR, CompactionState.SUCCEEDED)); Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=bbb", CompactionType.MINOR, CompactionState.SUCCEEDED)); - Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=null", CompactionType.MINOR, CompactionState.SUCCEEDED)); + Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=__HIVE_DEFAULT_PARTITION__", CompactionType.MINOR, CompactionState.SUCCEEDED)); // Additional compaction should be initiated for all partitions from past partition specs Assert.assertTrue(isCompactExist(rsp, null, CompactionType.MINOR, CompactionState.SUCCEEDED)); diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index efdbebcf9b54..ff85a29bd341 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -419,6 +419,7 @@ iceberg.llap.query.files=\ vectorized_iceberg_read_parquet.q iceberg.llap.query.compactor.files=\ + iceberg_compaction_colstats_compute.q,\ iceberg_major_compaction_partition_evolution.q,\ iceberg_major_compaction_partition_evolution2.q,\ iceberg_major_compaction_partition_evolution_ordered.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 0c4a56061cb7..a88f5275abfa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -153,7 +154,11 @@ private void getColumnsNoColumnPath(Table table, Partition partition, List valueMap.put(k, Longs.tryParse(v))); - numParts = sh.getPartitionNames(table).size(); + // the partition of no value holds what a table wrote before it had partitions, and no + // statement names it, so it's skipped here + numParts = (int) sh.getPartitionNames(table).stream() + .filter(name -> !DummyPartition.isVoid(name)) + .count(); } else { PartitionIterable partitions = new PartitionIterable(context.getDb(), table, null, @@ -280,11 +285,18 @@ private void addStatsForPartitionKeyColumn(Table table, List colStats, String colName, Map tableProps) throws HiveException { + if (table.isNonNative() && !StatsUtils.isPartitionStats(table, context.getConf())) { + // the table maintains table-level column statistics only, whose accuracy the table + // properties already reflect + colStats.addAll(context.getDb().getTableColumnStatistics(table, + Lists.newArrayList(colName.toLowerCase()), false)); + return; + } List parts = context.getDb().getPartitionNames(table, (short) -1); AggrStats aggrStats = context.getDb().getAggrColStatsFor(table, Lists.newArrayList(colName.toLowerCase()), parts, false); colStats.addAll(aggrStats.getColStats()); - + if (parts.size() == aggrStats.getPartsFound()) { StatsSetupConst.setColumnStatsState(tableProps, Lists.newArrayList(colName.toLowerCase())); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java index 58a829e3de22..9d30cf34c198 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.ddl.ShowUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -67,6 +68,10 @@ public int execute() throws HiveException { parts = context.getDb().getPartitionNames(tbl, desc.getLimit()); } + // the partition of no value holds what a table wrote before it had partitions, and no + // statement names it, so it's skipped here + parts = parts.stream().filter(name -> !DummyPartition.isVoid(name)).toList(); + // write the results in the file try (DataOutputStream outStream = ShowUtils.getOutputStream(new Path(desc.getResFile()), context)) { ShowPartitionsFormatter formatter = ShowPartitionsFormatter.getFormatter(context.getConf()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 221d2ac08f6f..b07470fb5374 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -680,10 +680,10 @@ public static Object[] populateVirtualColumnValues(ExecMapperContext ctx, vcValues[i] = new LongWritable(ctx.getIoCxt().getPositionDeleteInfo().getPartitionHash()); } break; - case PARTITION_PROJECTION: + case PARTITION_NAME: vcValues[i] = null; - if (ctx.getIoCxt().getPositionDeleteInfo() != null) { - vcValues[i] = new Text(ctx.getIoCxt().getPositionDeleteInfo().getPartitionProjection()); + if (ctx.getIoCxt().getPartitionName() != null) { + vcValues[i] = new Text(ctx.getIoCxt().getPartitionName()); } break; case FILE_PATH: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index bd9735100015..d7df1b039f98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -126,7 +126,10 @@ public int execute() { } private Table getTable(Hive db) throws SemanticException, HiveException { - return db.getTable(work.getFullTableName()); + Table table = db.getTable(work.getFullTableName()); + // a lookup by name carries no snapshot ref: restore the one the write targeted + table.setSnapshotRef(work.getTable().getSnapshotRef()); + return table; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java index ad438fd1b2e2..0dd8b3afb299 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java @@ -51,12 +51,15 @@ public class IOContext { /** * supports {@link org.apache.hadoop.hive.ql.metadata.VirtualColumn#ROWID} */ - private RecordIdentifier ri; + private RecordIdentifier ri; private boolean isDeletedRecord; private PositionDeleteInfo pdi; private RowLineageInfo rowLineageInfo; - public static enum Comparison { + // the partition of the rows the reader currently serves + private String partitionName; + + public enum Comparison { GREATER, LESS, EQUAL, @@ -199,6 +202,14 @@ public PositionDeleteInfo getPositionDeleteInfo() { return pdi; } + public void setPartitionName(String partitionName) { + this.partitionName = partitionName; + } + + public String getPartitionName() { + return partitionName; + } + public void parseRowLineageInfo(JobConf cconfiguration) { this.rowLineageInfo = RowLineageInfo.parseFromConf(cconfiguration); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java index f5e531496734..d3485a49f713 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java @@ -27,38 +27,32 @@ public class PositionDeleteInfo { private static final String CONF_KEY_PART_HASH = "hive.io.context.position.delete.partition.hash"; private static final String CONF_KEY_FILE_PATH = "hive.io.context.position.delete.file.path"; private static final String CONF_KEY_ROW_POSITION = "hive.io.context.position.delete.row.position"; - private static final String CONF_KEY_PARTITION_PROJECTION = "hive.io.context.position.delete.partition.projection"; public static PositionDeleteInfo parseFromConf(Configuration conf) { int specId = conf.getInt(CONF_KEY_SPEC_ID, -1); long partHash = conf.getLong(CONF_KEY_PART_HASH, -1); String filePath = conf.get(CONF_KEY_FILE_PATH); long rowPos = conf.getLong(CONF_KEY_ROW_POSITION, -1); - String partitionProjection = conf.get(CONF_KEY_PARTITION_PROJECTION); - return new PositionDeleteInfo(specId, partHash, filePath, rowPos, partitionProjection); + return new PositionDeleteInfo(specId, partHash, filePath, rowPos); } - public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos, - String partitionProjection) { + public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos) { conf.setInt(CONF_KEY_SPEC_ID, specId); conf.setLong(CONF_KEY_PART_HASH, partHash); conf.set(CONF_KEY_FILE_PATH, filePath); conf.setLong(CONF_KEY_ROW_POSITION, filePos); - conf.set(CONF_KEY_PARTITION_PROJECTION, partitionProjection); } private final int specId; private final long partitionHash; private final String filePath; private final long filePos; - private final String partitionProjection; - public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos, String partitionProjection) { + public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos) { this.specId = specId; this.partitionHash = partitionHash; this.filePath = filePath; this.filePos = filePos; - this.partitionProjection = partitionProjection; } public int getSpecId() { @@ -76,8 +70,4 @@ public String getFilePath() { public long getFilePos() { return filePos; } - - public String getPartitionProjection() { - return partitionProjection; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 03e7e5006474..236f6f3095f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -458,9 +458,11 @@ private void checkEndOfRowGroup() throws IOException { if(!colsToInclude.isEmpty()) { for (int i = 0; i < types.size(); ++i) { columnReaders[i] = - buildVectorizedParquetReader(columnTypesList.get(colsToInclude.get(i)), types.get(i), + buildVectorizedParquetReader( + columnTypesList.get(colsToInclude.get(i)), types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, 0, 0); + legacyConversionEnabled, 0, 0 + ); } } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index e4b3bb9eb836..4d55c8a7f9ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -6310,6 +6310,12 @@ public List getTableColumnStatistics( if (tbl.isNonNative() && tbl.getStorageHandler().canProvideColStatistics(tbl)) { return tbl.getStorageHandler().getColStatistics(tbl, colNames); } + if (tbl.isNonNative() && (tbl.getStorageHandler().canSetColStatistics(tbl) || + tbl.getSnapshotRef() != null)) { + // the handler owns the table's statistics, or the read is of a branch: the metastore's + // single set describes neither + return Collections.emptyList(); + } if (checkTransactional) { AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl); retv = getMSC().getTableColumnStatistics(tbl.getDbName(), tbl.getTableName(), colNames, @@ -6364,6 +6370,12 @@ public AggrStats getAggrColStatsFor(Table tbl, if (tbl.isNonNative() && tbl.getStorageHandler().canProvideColStatistics(tbl)) { return tbl.getStorageHandler().getAggrColStatsFor(tbl, colNames, partName); } + if (tbl.isNonNative() && (tbl.getStorageHandler().canSetColStatistics(tbl) || + tbl.getSnapshotRef() != null)) { + // the handler owns the table's statistics, or the read is of a branch: the metastore's + // single set describes neither + return new AggrStats(new ArrayList<>(), 0); + } if (checkTransactional) { AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl); writeIdList = tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index ad18304fa72b..e6c6fe0206c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.common.type.SnapshotContext; @@ -72,15 +73,18 @@ import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils; import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCommitter; import org.apache.hadoop.mapred.OutputFormat; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Function; /** * HiveStorageHandler defines a pluggable interface for adding @@ -328,13 +332,16 @@ default Map> getAggrBasicStatsFor(org.apache.hadoop. } /** - * Set column stats for non-native tables + * Persists the column statistics a gather computed. They are pulled batch by batch, so the whole + * of a large table's statistics is never held at once. * @param table table object - * @param colStats list of ColumnStatistics objects - * @return true if operation is successful + * @param colStats the computed statistics, one entry for the table or one per partition + * @return whether the stored statistics now describe the table */ - default boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, List colStats) { - return false; + default boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, + Iterator colStats) { + throw new UnsupportedOperationException( + this.getClass().getName() + " does not support column statistics"); } /** @@ -355,6 +362,29 @@ default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table tab return false; } + /** + * Check if the storage handler can set col statistics of the given granularity. A handler keeps + * them either per partition or for the table as a whole, and one that keeps any says which. + * @param table table object + * @param partitionLevel whether the statistics asked about are the per partition ones + * @return true if the storage handler can set col statistics of that granularity + */ + default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, boolean partitionLevel) { + return false; + } + + /** + * Whether the column statistics the handler holds still describe the table, so that a query may + * be answered from them rather than by reading the data. The metastore's accuracy marker only + * records what Hive itself wrote, while a handler's table may be written by other engines. + * @param table table object + * @param colName the column being asked about + * @return true if the statistics still describe the table + */ + default boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table table, String colName) { + return StatsSetupConst.areColumnStatsUptoDate(table.getParameters(), colName); + } + /** * Returns the row count of the table, letting queries like count(1) be answered from statistics. * @param hmsTable table object @@ -604,10 +634,20 @@ default List getPartitionTransformSpec(org.apache.hadoop.hive.ql. return null; } - default Map> getPartitionTransformSpecs(org.apache.hadoop.hive.ql.metadata.Table table) { - return null; + /** + * Returns a function naming the partition a row belongs to. Statistics join on this name, so a + * handler must derive it the way it derives the names of the partitions it writes. Deriving one + * may cost as much as writing a row, so the function is asked for once for all the rows an + * inspector reads, and is neither reentrant nor thread-safe. + * @param table the HMS table, must be non-null + * @param inspector the inspector of a row holding the columns the partitioning is derived from + */ + default Function partitionNameResolver( + org.apache.hadoop.hive.ql.metadata.Table table, StructObjectInspector inspector) { + throw new UnsupportedOperationException(getClass().getName() + " does not name partitions"); } + /** * Creates a DynamicPartitionCtx instance that will be set up by the storage handler itself. Useful for non-native * tables where partitions are not handled by Hive, and sorting is required in a custom way before writing the table. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 2592113614cc..3cdfba01c97a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -285,7 +285,9 @@ public void checkValidity(Configuration conf) throws HiveException { // check for validity validateName(conf); - if (getCols().isEmpty()) { + if (getCols().isEmpty() && !hasNonNativePartitionSupport()) { + // a non-native table's data-column view excludes the handler partition columns and is + // legitimately empty when every column is a partition transform source throw new HiveException("at least one column must be specified for the table"); } validateColumns(getCols(), getPartCols(), DDLUtils.isIcebergTable(this)); @@ -1463,16 +1465,21 @@ public List getVirtualColumns() { List virtualColumns = new ArrayList<>(); if (!isNonNative()) { virtualColumns.addAll(VirtualColumn.getRegistry()); + return virtualColumns; } - if (isNonNative() && AcidUtils.isNonNativeAcidTable(this)) { + if (AcidUtils.isNonNativeAcidTable(this)) { virtualColumns.addAll(getStorageHandler().acidVirtualColumns()); } - if (isNonNative() && getStorageHandler().areSnapshotsSupported() && - isBlank(getMetaTable())) { + if (!isBlank(getMetaTable())) { + return virtualColumns; + } + if (hasNonNativePartitionSupport()) { + virtualColumns.add(VirtualColumn.PARTITION_NAME); + } + if (getStorageHandler().areSnapshotsSupported()) { virtualColumns.add(VirtualColumn.SNAPSHOT_ID); } - if (isNonNative() && getStorageHandler().supportsRowLineage(getTTable().getParameters()) && - isBlank(getMetaTable())) { + if (getStorageHandler().supportsRowLineage(getTTable().getParameters())) { virtualColumns.add(VirtualColumn.ROW_LINEAGE_ID); virtualColumns.add(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index 227dda27d9d2..4fd718af61be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -57,7 +57,7 @@ public enum VirtualColumn { FILE_PATH("FILE__PATH", TypeInfoFactory.stringTypeInfo), ROW_POSITION("ROW__POSITION", TypeInfoFactory.longTypeInfo), SNAPSHOT_ID("SNAPSHOT__ID", TypeInfoFactory.longTypeInfo), - PARTITION_PROJECTION("PARTITION__PROJECTION", TypeInfoFactory.stringTypeInfo), + PARTITION_NAME("PARTITION__NAME", TypeInfoFactory.stringTypeInfo), ROW_LINEAGE_ID("ROW__LINEAGE__ID", TypeInfoFactory.longTypeInfo), LAST_UPDATED_SEQUENCE_NUMBER("LAST__UPDATED__SEQUENCE__NUMBER", TypeInfoFactory.longTypeInfo), @@ -74,7 +74,8 @@ public enum VirtualColumn { ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName(), ROWISDELETED.getName(), PARTITION_SPEC_ID.getName(), PARTITION_HASH.getName(), FILE_PATH.getName(), ROW_POSITION.getName(), - PARTITION_PROJECTION.getName(), ROW_LINEAGE_ID.getName(), LAST_UPDATED_SEQUENCE_NUMBER.getName()); + PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), + LAST_UPDATED_SEQUENCE_NUMBER.getName()); public static final ImmutableMap VIRTUAL_COLUMN_NAME_MAP = new ImmutableMap.Builder().putAll(getColumnNameMap()).build(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index fe477303cadb..fa5317271678 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1537,6 +1537,11 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, } } + // the lookups above carry no snapshot ref: take it from the table the file sink writes to + if (table != null && nd.getConf().getTable() != null) { + table.setSnapshotRef(nd.getConf().getTable().getSnapshotRef()); + } + StatsWork columnStatsWork = new StatsWork(table, statsWork, hconf); columnStatsWork.collectStatsFromAggregator(nd.getConf()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index bff675b0d98f..022982f1740d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -21,13 +21,15 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -49,7 +51,6 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.SemanticRule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -79,14 +80,14 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.List; import java.util.Map; import java.util.Set; @@ -215,7 +216,7 @@ private StatType getType(String origType) { return StatType.Unsupported; } - private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { + private Long getNullCountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : @@ -235,6 +236,22 @@ private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { } } + /** + * The statistics of no rows: nothing counted, no low or high value. The branches below already + * fold that to the right answer - zero for a count, NULL for a min or a max. + */ + private static ColumnStatisticsData emptyColStats(StatType type) { + return switch (type) { + case Integer -> ColumnStatisticsData.longStats(new LongColumnStatsData()); + case Double -> ColumnStatisticsData.doubleStats(new DoubleColumnStatsData()); + case String -> ColumnStatisticsData.stringStats(new StringColumnStatsData()); + case Boolean -> ColumnStatisticsData.booleanStats(new BooleanColumnStatsData()); + case Binary -> ColumnStatisticsData.binaryStats(new BinaryColumnStatsData()); + case Date -> ColumnStatisticsData.dateStats(new DateColumnStatsData()); + default -> null; + }; + } + private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); int numCols = gbyDesc.getOutputColumnNames().size(); @@ -390,8 +407,13 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { List oneRow = new ArrayList(); - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(pctx.getConf(), tbl); + // Every aggregate of one query asks the same partitions about a column of the same table, + // and the statistics of one partition carry every column, so asking once for all of them + // reads what a thousand aggregates would have read a thousand times. + PrunedPartitionList prunedList = tbl.isPartitioned() ? + pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) : null; + ScanColStats scanColStats = + new ScanColStats(hive, tbl, aggregateColumns(pgbyOp, exprMap), prunedList); for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { if (aggr.getDistinct()) { @@ -470,27 +492,19 @@ else if (udaf instanceof GenericUDAFCount) { String colName = desc.getColumn(); StatType type = getType(desc.getTypeString()); if (!tbl.isPartitioned()) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(tbl, tbl.getParameters())) { - Logger.debug("Stats for table : " + tbl.getTableName() + " are not up to date."); + // asked of the table, not read off its parameters: those describe the current + // snapshot, while the column statistics below answer for the one this scan reads + Long tableRowCnt = getRowCnt(tsOp, tbl); + if (tableRowCnt == null) { + Logger.debug("No exact row count for table : " + tbl.getTableName()); return null; } - rowCnt = Long.valueOf(tbl.getProperty(StatsSetupConst.ROW_COUNT)); - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; - } - - List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + rowCnt = tableRowCnt; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside } - Long nullCnt = getNullcountFor(type, stats.get(0).getStatsData()); + Long nullCnt = getNullCountFor(type, statData); if (null == nullCnt) { Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + "metadata optimizer for column : " + colName); @@ -499,35 +513,37 @@ else if (udaf instanceof GenericUDAFCount) { rowCnt -= nullCnt; } } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) - .getPartitions(); - for (Partition part : parts) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { - Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); + if (tbl.isNonNative()) { + // a handler holds no partition parameters, so its row counts are asked of the + // table, which is told the partitions this scan was pruned to + Long handlerRowCnt = getRowCnt(tsOp, tbl); + if (handlerRowCnt == null) { + Logger.debug("No exact row count for table : " + tbl.getTableName()); return null; } - long partRowCnt = Long.parseLong(part.getParameters().get( - StatsSetupConst.ROW_COUNT)); - rowCnt += partRowCnt; + rowCnt = handlerRowCnt; + } else { + for (Partition part : prunedList.getPartitions()) { + if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { + Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); + return null; + } + long partRowCnt = Long.parseLong(part.getParameters().get( + StatsSetupConst.ROW_COUNT)); + rowCnt += partRowCnt; + } } - Collection> result = verifyAndGetPartColumnStats(hive, - tbl, colName, parts); - if (result == null) { + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) - return null; - Long nullCnt = getNullcountFor(type, statData); - if (nullCnt == null) { - Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " - + "metadata optimizer for column : " + colName); - return null; - } else { - rowCnt -= nullCnt; - } + Long nullCnt = getNullCountFor(type, statData); + if (nullCnt == null) { + Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } + rowCnt -= nullCnt; } } oneRow.add(rowCnt); @@ -535,284 +551,67 @@ else if (udaf instanceof GenericUDAFCount) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); - if(!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside + } + String name = colDesc.getTypeString().toUpperCase(); + switch (type) { + case Integer: { + LongColumnStatsData lstats = statData.getLongStats(); + oneRow.add(lstats.isSetHighValue() ? + LongSubType.valueOf(name).cast(lstats.getHighValue()) : null); + break; } - - List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + case Double: { + DoubleColumnStatsData dstats = statData.getDoubleStats(); + oneRow.add(dstats.isSetHighValue() ? + DoubleSubType.valueOf(name).cast(dstats.getHighValue()) : null); + break; } - ColumnStatisticsData statData = stats.get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetHighValue()) { - oneRow.add(subType.cast(lstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetHighValue()) { - oneRow.add(subType.cast(dstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetHighValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getHighValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: - // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - } - } else { - Set parts = pctx.getPrunedPartitions( - tsOp.getConf().getAlias(), tsOp).getPartitions(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetHighValue()) { - continue; - } - long curVal = lstats.getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(maxVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetHighValue()) { - continue; - } - double curVal = statData.getDoubleStats().getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - case Date: { - Long maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetHighValue()) { - continue; - } - long curVal = dstats.getHighValue().getDaysSinceEpoch(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(DateSubType.DAYS.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - default: - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; + case Date: { + DateColumnStatsData dstats = statData.getDateStats(); + oneRow.add(dstats.isSetHighValue() ? + DateSubType.DAYS.cast(dstats.getHighValue().getDaysSinceEpoch()) : null); + break; } + default: + Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } } else if (udaf instanceof GenericUDAFMin) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); - if (!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside + } + String name = colDesc.getTypeString().toUpperCase(); + switch (type) { + case Integer: { + LongColumnStatsData lstats = statData.getLongStats(); + oneRow.add(lstats.isSetLowValue() ? + LongSubType.valueOf(name).cast(lstats.getLowValue()) : null); + break; } - ColumnStatisticsData statData = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null) - .get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetLowValue()) { - oneRow.add(subType.cast(lstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetLowValue()) { - oneRow.add(subType.cast(dstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetLowValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getLowValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; + case Double: { + DoubleColumnStatsData dstats = statData.getDoubleStats(); + oneRow.add(dstats.isSetLowValue() ? + DoubleSubType.valueOf(name).cast(dstats.getLowValue()) : null); + break; } - } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions(); - String name = colDesc.getTypeString().toUpperCase(); - switch(type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetLowValue()) { - continue; - } - long curVal = lstats.getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetLowValue()) { - continue; - } - double curVal = statData.getDoubleStats().getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Date: { - Long minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetLowValue()) { - continue; - } - long curVal = dstats.getLowValue().getDaysSinceEpoch(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(DateSubType.DAYS.cast(minVal)); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - + case Date: { + DateColumnStatsData dstats = statData.getDateStats(); + oneRow.add(dstats.isSetLowValue() ? + DateSubType.DAYS.cast(dstats.getLowValue().getDaysSinceEpoch()) : null); + break; } + default: + Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } } else { // Unsupported aggregation. Logger.debug("Unsupported aggregation for metadata optimizer: " @@ -899,39 +698,130 @@ else if (udaf instanceof GenericUDAFCount) { } } - private ColumnStatisticsData validateSingleColStat(List statObj) { - if (statObj.size() > 1) { - Logger.error("More than one stat for a single column!"); - return null; - } else if (statObj.isEmpty()) { - Logger.debug("No stats for some partition and column"); - return null; - } - return statObj.get(0).getStatsData(); + + /** The columns the aggregates read, which are the ones statistics have to be fetched for. */ + private static List aggregateColumns(GroupByOperator pgbyOp, Map exprMap) { + return pgbyOp.getConf().getAggregators().stream() + .filter(aggr -> !aggr.getParameters().isEmpty()) + .map(aggr -> aggr.getParameters().get(0)) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> exprMap.get(((ExprNodeColumnDesc) desc).getColumn())) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> ((ExprNodeColumnDesc) desc).getColumn()) + .distinct() + .collect(Collectors.toList()); } - private Collection> verifyAndGetPartColumnStats( - Hive hive, Table tbl, String colName, Set parts) throws TException, LockException { - List partNames = new ArrayList(parts.size()); - for (Partition part : parts) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { - Logger.debug("Stats for part : " + part.getSpec() + " column " + colName - + " are not up to date."); + /** + * The statistics of the columns a scan's aggregates read, fetched for every column when the + * first of them asks. An aggregate this rewrite cannot answer declines the query before + * asking, so a query holding one pays for nothing. Answers for a scan of a partitioned table. + */ + private static final class ScanColStats { + private final Hive hive; + private final Table tbl; + private final List colNames; + private final PrunedPartitionList prunedList; + private Map colStatsByName; + private boolean fetched; + + ScanColStats(Hive hive, Table tbl, List colNames, PrunedPartitionList prunedList) { + this.hive = hive; + this.tbl = tbl; + this.colNames = colNames; + this.prunedList = prunedList; + } + + /** + * One column's statistics. A scan pruned to no partitions reads no rows, and the statistics + * of no rows are the empty ones: nothing counted, and no least or greatest to name. + */ + ColumnStatisticsData statsFor(String colName, StatType type) throws HiveException { + if (prunedList != null && prunedList.getPartitions().isEmpty()) { + return emptyColStats(type); + } + if (!fetched) { + fetched = true; + colStatsByName = prunedList == null ? tableColStats() : verifyAndFetch(); + } + ColumnStatisticsObj stat = colStatsByName == null ? null : colStatsByName.get(colName); + if (stat == null) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); return null; } - partNames.add(part.getName()); + return stat.getStatsData(); } - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(hive.getConf(), tbl); - - Map> result = hive.getMSC().getPartitionColumnStatistics( - tbl.getDbName(), tbl.getTableName(), partNames, Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (result.size() != parts.size()) { - Logger.debug("Received " + result.size() + " stats for " + parts.size() + " partitions"); - return null; + + /** + * Whether the table's own statistics answer for this scan: it keeps them for the table as + * a whole, and the scan reads every partition. They then describe exactly the rows read. + */ + private boolean answeredByTheTable() { + return !StatsUtils.isPartitionStats(tbl, hive.getConf()) && + prunedList.getReferredPartCols().isEmpty() && !prunedList.hasUnknownPartitions(); + } + + /** The table's own statistics, taken only while they still describe it. */ + private Map tableColStats() throws HiveException { + for (String colName : colNames) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { + Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName + + " are not up to date."); + return null; + } + } + return byColumnName(hive.getTableColumnStatistics(tbl, colNames, true)); } - return result.values(); + + /** What the scan's partitions hold for every column asked about, or null to decline. */ + private Map verifyAndFetch() throws HiveException { + Set parts = prunedList.getPartitions(); + List partNames = new ArrayList<>(parts.size()); + // a storage handler holds no partition parameters, and one kept per partition describes no + // partition in particular: whether each still describes itself is answered by the aggregate + // below, which is told the partitions this query pruned to + if (tbl.isNonNative()) { + if (!StatsUtils.checkCanProvideColumnStats(tbl)) { + Logger.debug("Table : " + tbl.getTableName() + " provides no column statistics."); + return null; + } + if (answeredByTheTable()) { + return tableColStats(); + } + parts.forEach(part -> partNames.add(part.getName())); + } else { + for (Partition part : parts) { + for (String colName : colNames) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering( + part.getTable(), part.getParameters(), colName)) { + Logger.debug("Stats for part : " + part.getSpec() + " column " + colName + + " are not up to date."); + return null; + } + } + partNames.add(part.getName()); + } + } + // Aggregated rather than per partition: the callers fold these with min, max or a sum, so + // merging first gives the same answer. It also reaches a handler's own statistics, which + // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. + AggrStats aggrStats = hive.getAggrColStatsFor(tbl, colNames, partNames, true); + if (aggrStats == null || aggrStats.getColStats() == null) { + Logger.debug("No stats for " + tbl.getTableName() + " columns " + colNames); + return null; + } + if (aggrStats.getPartsFound() != parts.size()) { + // a partition whose statistics are missing would leave the answer describing a subset + Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); + return null; + } + return byColumnName(aggrStats.getColStats()); + } + } + + private static Map byColumnName(List colStats) { + return colStats.stream().collect( + Collectors.toMap(ColumnStatisticsObj::getColName, Function.identity(), (first, second) -> second)); } private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 512337c11fdb..12c43f7f6e8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -309,7 +309,7 @@ public class Vectorizer implements PhysicalPlanResolver { VirtualColumn.PARTITION_HASH, VirtualColumn.FILE_PATH, VirtualColumn.ROW_POSITION, - VirtualColumn.PARTITION_PROJECTION, + VirtualColumn.PARTITION_NAME, VirtualColumn.ROW_LINEAGE_ID, VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); private HiveConf hiveConf; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java index 10f90c55e8ea..6c820d303f25 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java @@ -22,9 +22,9 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.session.SessionState; + +import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getQualifiedTableName; /** * Utilities for semantic analyzers. @@ -47,10 +47,9 @@ public static boolean isPartitionLevelStats(ASTNode tree) { } public static Table getTable(ASTNode tree, BaseSemanticAnalyzer sa) throws SemanticException { - String tableName = ColumnStatsSemanticAnalyzer.getUnescapedName((ASTNode) tree.getChild(0).getChild(0)); - String currentDb = SessionState.get().getCurrentDatabase(); - String [] names = Utilities.getDbTableName(currentDb, tableName); - return sa.getTable(names[0], names[1], true); + return sa.getTable( + getQualifiedTableName((ASTNode) tree.getChild(0).getChild(0)), + true); } public static Map getPartKeyValuePairsFromAST(Table tbl, ASTNode tree, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index fee103ec4f09..bf851fa8e5cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -68,9 +68,10 @@ public class ColumnStatsAutoGatherContext { private final List partitionColumns; private boolean isInsertInto; private Table tbl; - private List partTransformSpec; private Map partSpec; private Context origCtx; + + private boolean isTableLevel = true; public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, Operator op, Table tbl, Map partSpec, @@ -124,18 +125,22 @@ public void insertAnalyzePipeline() throws SemanticException { */ public void insertTableValuesAnalyzePipeline() throws SemanticException { // Instead of starting from analyze statement, we just generate the Select plan - boolean isPartitionStats = StatsUtils.isPartitionStats(tbl, conf); - if (isPartitionStats) { + // the table is not created yet, so only the CREATE statement says how it partitions its rows + List partTransformSpec = tbl.hasNonNativePartitionSupport() ? + TransformSpec.fromQueryState(conf) : null; + + boolean isPartitionStats = partTransformSpec != null ? + StatsUtils.isPartitionStatsEnabled(tbl, conf) : StatsUtils.isPartitionStats(tbl, conf); + + if (isPartitionStats && partTransformSpec == null) { partSpec = new HashMap<>(); List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); partKeys.forEach(k -> partSpec.put(k, null)); - - if (tbl.hasNonNativePartitionSupport()) { - partTransformSpec = tbl.getStorageHandler().getPartitionTransformSpec(tbl); - } } - String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery( - tbl, conf, partTransformSpec, partSpec, isPartitionStats); + isTableLevel = !isPartitionStats; + + String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, conf, + partTransformSpec, partSpec, isPartitionStats); insertAnalyzePipeline(command, true); } @@ -177,6 +182,9 @@ private Operator genSelOp(String command, boolean rewritten, Context origCtx) if (rewritten) { // Create the context object that is needed to store the column stats this.analyzeRewrite = ColumnStatsSemanticAnalyzer.genAnalyzeRewriteContext(conf, tbl); + // a table the CREATE has yet to make cannot be asked how it partitions its rows, so the + // scope the statement was written for is the one that holds + this.analyzeRewrite.setTblLvl(isTableLevel); // The analyze statement has already been rewritten, we just need to create the AST // and the corresponding semantic analyzer diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index b82b21068a6c..20b48f40208f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hive.ql.parse; import static org.apache.hadoop.hive.ql.metadata.HiveUtils.unparseIdentifier; -import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_SPEC_ID; +import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_NAME; import com.google.common.base.Preconditions; import java.util.ArrayList; @@ -31,7 +31,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import com.google.common.collect.Maps; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -44,13 +43,16 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsField; import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsType; import org.apache.hadoop.hive.ql.stats.StatsUtils; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -66,6 +68,12 @@ * */ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer { + /** + * Set while an ANALYZE that named partitions is compiled: its statistics stand for those alone, + * so what the table keeps for the rest has to survive them. + */ + public static final String ANALYZE_PARTITION = "hive.stats.analyze.partition"; + private static final Logger LOG = LoggerFactory .getLogger(ColumnStatsSemanticAnalyzer.class); private static final LogHelper CONSOLE = new LogHelper(LOG); @@ -147,8 +155,8 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } try { // for static partition, it may not exist when HIVE_STATS_COL_AUTOGATHER is - // set to true - if (context == null && partValsSpecified > 0) { + // set to true. A table partitioned outside the metastore keeps no partition there to find. + if (context == null && partValsSpecified > 0 && !tbl.hasNonNativePartitionSupport()) { if ((partValsSpecified == tbl.getPartitionKeys().size()) && (db.getPartition(tbl, partSpec, false, null, false) == null)) { throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() @@ -161,7 +169,7 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } // User might have only specified partial list of partition keys, in which case add other partition keys in partSpec - List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); + List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartCols()); for (String partKey : partKeys) { if (!partSpec.containsKey(partKey)) { partSpec.put(partKey, null); @@ -176,37 +184,83 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } } - private static CharSequence genPartitionClause(Table tbl, List partTransformSpec, int specId, - Map partSpec, HiveConf conf) { - boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); - - StringBuilder whereClause = new StringBuilder(" where ").append( + /** The predicate naming the partitions an ANALYZE was pointed at, empty when it named none. */ + private static CharSequence genPartitionPredicate(Table tbl, Map partSpec, HiveConf conf) { + if (partSpec == null || partSpec.values().stream().noneMatch(Objects::nonNull)) { + return ""; + } + return new StringBuilder(" where ").append( partSpec.entrySet().stream() .filter(part -> part.getValue() != null) - .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + genPartValueString(getColTypeOf(tbl, part.getKey()), part.getValue())) .collect(Collectors.joining(" and ")) ); + } - if (specId >= 0) { - whereClause.append((predPresent) ? " and " : "") - .append(unparseIdentifier(PARTITION_SPEC_ID.getName(), conf) + "=" + specId); - predPresent = true; - } + private static CharSequence genPartitionClause(Table tbl, Map partSpec, HiveConf conf) { + boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); - StringBuilder groupByClause = new StringBuilder(" group by ").append(( - (partTransformSpec != null) ? - partTransformSpec.stream().map(spec -> spec.toHiveExpr(conf)) : - tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) - ) - .collect(Collectors.joining(", ")) + StringBuilder whereClause = new StringBuilder(genPartitionPredicate(tbl, partSpec, conf)); + + StringBuilder groupByClause = new StringBuilder(" group by ").append( + tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) + .collect(Collectors.joining(", ")) ); // attach the predicate and group by to the return clause return predPresent ? whereClause.append(groupByClause) : groupByClause; } + /** + * Narrows the scan to the partitions of the current spec the statement named, which it has. A value + * predicate names rows, not partitions, so on its own it reaches into a partition of any spec + * holding a row that carries the value, and a group formed of those would describe that partition + * by part of what it holds. The partitions of one spec share its transforms, so applying them to + * the value named holds every row of each partition named, and prunes what the scan reads. + */ + private static CharSequence genNamedPartitionClause(Table tbl, Map named, HiveConf conf) + throws SemanticException { + List partitions = tbl.getStorageHandler().getPartitions(tbl, named, true); + if (partitions.isEmpty()) { + throw new SemanticException( + ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() + " : " + named); + } + String scanned = partitions.stream() + .map(Partition::getName) + .map(name -> genPartValueString(serdeConstants.STRING_TYPE_NAME, name)) + .collect(Collectors.joining(", ", + " where " + unparseIdentifier(PARTITION_NAME.getName(), conf) + " in (", ")")); + String pruned = partitionValuePredicate(tbl, named, conf); + return pruned.isEmpty() ? scanned : scanned + " and " + pruned; + } + /** The partition columns the statement gave a value, which are the ones it named. */ + private static Map namedPartitionValues(Map partSpec) { + return partSpec == null ? Collections.emptyMap() : + partSpec.entrySet().stream() + .filter(part -> part.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * What the named partitions hold, as their spec decides it: the transform of a partition field + * takes the same value for every row of the partition, so comparing it against the transform of + * the value named keeps all of them. + */ + private static String partitionValuePredicate(Table tbl, Map named, HiveConf conf) { + return tbl.getStorageHandler().getPartitionTransformSpec(tbl).stream() + .filter(spec -> named.containsKey(spec.getColumnName())) + .map(spec -> { + String value = named.get(spec.getColumnName()); + // the partition of no value is named, not valued, and a transform of nothing is nothing + return ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal.equals(value) ? + spec.toHiveExpr(conf) + " is null" : + spec.toHiveExpr(conf) + " = " + spec.toHiveExpr( + genPartValueString(getColTypeOf(tbl, spec.getColumnName()), value)); + }) + .collect(Collectors.joining(" and ")); + } private static String getColTypeOf(Table tbl, String partKey) { for (FieldSchema fs : tbl.getPartCols()) { @@ -261,10 +315,10 @@ protected static List getFieldSchemasByColName(Table tbl, List partTransformSpec, int specId, Map partSpec, - boolean isPartitionStats) { - String rewritten = genRewrittenQuery(tbl, columnSchemas, conf, partTransformSpec, specId, partSpec, - isPartitionStats, false); + List partTransformSpec, Map partSpec, boolean isPartitionStats) + throws SemanticException { + String rewritten = + genRewrittenQuery(tbl, columnSchemas, conf, partTransformSpec, partSpec, isPartitionStats, false); isRewritten = true; return rewritten; } @@ -273,16 +327,16 @@ private String genRewrittenQuery(FieldSchemas columnSchemas, HiveConf conf, * Generates a SQL statement that will compute the stats for all columns * included in the input table. */ - protected static String genRewrittenQuery(Table tbl, - HiveConf conf, List partTransformSpec, Map partSpec, - boolean isPartitionStats) { + protected static String genRewrittenQuery(Table tbl, HiveConf conf, + List partTransformSpec, Map partSpec, boolean isPartitionStats) + throws SemanticException { return ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, getStatsEligibleFieldSchemas(tbl), conf, - partTransformSpec, -1, partSpec, isPartitionStats, true); + partTransformSpec, partSpec, isPartitionStats, true); } - private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, - HiveConf conf, List partTransformSpec, int specId, Map partSpec, - boolean isPartitionStats, boolean useTableValues) { + private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, HiveConf conf, + List partTransformSpec, Map partSpec, boolean isPartitionStats, + boolean useTableValues) throws SemanticException { StringBuilder rewrittenQueryBuilder = new StringBuilder("select "); StringBuilder columnNamesBuilder = new StringBuilder(); @@ -298,11 +352,7 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, final String columnName = unparseIdentifier(columnSchema.getName(), conf); final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(columnSchema.getType()); - try { - genComputeStats(rewrittenQueryBuilder, conf, i, columnName, typeInfo); - } catch (SemanticException e) { - throw new RuntimeException(e); - } + genComputeStats(rewrittenQueryBuilder, conf, i, columnName, typeInfo); columnNamesBuilder.append(columnName); @@ -311,7 +361,14 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, } if (isPartitionStats) { - if (partTransformSpec == null) { + if (partTransformSpec != null) { + // a write produces one spec, so its rows can be grouped by the transforms themselves, + // and carry the values those were applied to for the table to name the group + rewrittenQueryBuilder.append(", ").append(TransformSpec.toSourceStruct(partTransformSpec, conf)); + } else if (tbl.hasNonNativePartitionSupport()) { + // group every row by its read-side partition name + rewrittenQueryBuilder.append(", ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + } else { for (FieldSchema fs : tbl.getPartCols()) { String identifier = unparseIdentifier(fs.getName(), conf); rewrittenQueryBuilder.append(", ").append(identifier); @@ -321,9 +378,6 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, .append(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()).toString()) .append(")"); } - } else { - rewrittenQueryBuilder.append(", ") - .append(TransformSpec.toNamedStruct(partTransformSpec, conf)); } } @@ -341,19 +395,31 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, .append(")"); } else { rewrittenQueryBuilder.append(unparseIdentifier(tbl.getDbName(), conf)) - .append(".") - .append(unparseIdentifier(tbl.getTableName(), conf)); - - if (tbl.getMetaTable() != null) { - rewrittenQueryBuilder.append(".") - .append(unparseIdentifier(tbl.getMetaTable(), conf)); + .append(".").append(unparseIdentifier(tbl.getTableName(), conf)); + + if (tbl.getSnapshotRef() != null) { + rewrittenQueryBuilder.append(".").append(unparseIdentifier(tbl.getSnapshotRef(), conf)); } } // If partition level statistics is requested, add predicate and group by as needed to rewritten // query if (isPartitionStats) { - rewrittenQueryBuilder.append(genPartitionClause(tbl, partTransformSpec, specId, partSpec, conf)); + if (partTransformSpec != null) { + rewrittenQueryBuilder.append("\ngroup by ").append(partTransformSpec.stream() + .map(spec -> spec.toHiveExpr(conf)).collect(Collectors.joining(", "))); + } else if (!tbl.hasNonNativePartitionSupport()) { + rewrittenQueryBuilder.append(genPartitionClause(tbl, partSpec, conf)); + } else { + // the name groups rows of every spec alike, so the scan is held to the partitions the + // statement named by naming them too + Map named = namedPartitionValues(partSpec); + if (!named.isEmpty()) { + rewrittenQueryBuilder.append(genNamedPartitionClause(tbl, named, conf)); + } + rewrittenQueryBuilder.append("\ngroup by ") + .append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + } } String rewrittenQuery = rewrittenQueryBuilder.toString(); @@ -525,7 +591,7 @@ private static void appendKllSketch(StringBuilder rewrittenQueryBuilder, HiveCon private static void appendBitVector(StringBuilder rewrittenQueryBuilder, HiveConf conf, String columnName) throws SemanticException { - String func = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_NDV_ALGO).toLowerCase(); + String func = HiveConf.getVar(conf, ConfVars.HIVE_STATS_NDV_ALGO).toLowerCase(); if ("hll".equals(func)) { rewrittenQueryBuilder .append("compute_bit_vector_hll(") @@ -627,16 +693,18 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { */ if (shouldRewrite(ast)) { tbl = AnalyzeCommandUtils.getTable(ast, this); + + if (tbl.getMetaTable() != null) { + throw new SemanticException("Cannot analyze metadata table: " + tbl.getMetaTable()); + } originalTree = ast; boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) || StatsUtils.isPartitionStats(tbl, conf); - // a partition-scoped column statistics ANALYZE cannot be honored for non-native tables: the stats - // of all partitions are rewritten as a whole, so it would drop every other partition's statistics - // (table-level ANALYZE of a partitioned table still computes all partitions - only the explicit - // partition spec is rejected; the auto-gather path merges instead and stays unaffected) - validateUnsupportedPartitionClause(tbl, AnalyzeCommandUtils.isPartitionLevelStats(ast)); + // naming partitions asks for the statistics of those alone, which a table keeping one set for + // itself has nowhere to put + validateUnsupportedPartitionClause(tbl, + AnalyzeCommandUtils.isPartitionLevelStats(ast) && !StatsUtils.isPartitionStats(tbl, conf)); - Map> partTransformSpecs = Collections.singletonMap(-1, null); Map partSpec = (isPartitionStats) ? AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf) : null; @@ -644,18 +712,13 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { if (isPartitionStats) { handlePartialPartitionSpec(partSpec, null); - if (tbl.hasNonNativePartitionSupport()) { - partTransformSpecs = tbl.getStorageHandler().getPartitionTransformSpecs(tbl); - } } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - - rewrittenQuery = String.join(" union all ", - Maps.transformEntries(partTransformSpecs, (specId, partTransformSpec) -> - genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, specId, partSpec, isPartitionStats)) - .values()); - + if (partSpec != null && partSpec.values().stream().anyMatch(Objects::nonNull)) { + SessionStateUtil.addResourceOrThrow(conf, ANALYZE_PARTITION, Boolean.TRUE); + } + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, null, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); } else { // Not an analyze table column compute statistics statement - don't do any rewrites @@ -705,7 +768,6 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) || StatsUtils.isPartitionStats(tbl, conf); - List partTransformSpec = null; Map partSpec = null; @@ -715,14 +777,14 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) partSpec = AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf); handlePartialPartitionSpec(partSpec, context); if (tbl.hasNonNativePartitionSupport()) { + // a write produces one spec, so the transforms group its rows partTransformSpec = tbl.getStorageHandler().getPartitionTransformSpec(tbl); } } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, -1, - partSpec, isPartitionStats); + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); return rewrittenTree; @@ -753,7 +815,7 @@ AnalyzeRewriteContext getAnalyzeRewriteContext() { static AnalyzeRewriteContext genAnalyzeRewriteContext(HiveConf conf, Table tbl) { AnalyzeRewriteContext analyzeRewrite = new AnalyzeRewriteContext(); analyzeRewrite.setTableName(tbl.getFullyQualifiedName()); - analyzeRewrite.setTblLvl(!(conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && tbl.isPartitioned())); + analyzeRewrite.setTblLvl(!StatsUtils.isPartitionStats(tbl, conf)); analyzeRewrite.setFieldSchemas(getStatsEligibleFieldSchemas(tbl)); return analyzeRewrite; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6bcbd346b235..c370110a68cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -266,6 +266,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; @@ -8848,6 +8849,13 @@ private void createPreInsertDesc(Table table, boolean overwrite) { private void genAutoColumnStatsGatheringPipeline(Table table, Map partSpec, Operator curr, boolean isInsertInto, boolean useTableValueConstructor) throws SemanticException { + if (isInsertInto && table.hasNonNativePartitionSupport() && StatsUtils.isPartitionStats(table, conf)) { + // this table keeps its column statistics per partition, and an insert reaches too few of them + // to pay for grouping the gather by partition; they stand until something covers the table + LOG.debug("Skipping column stats autogather for insert into partition-level table {}", + table.getTableName()); + return; + } LOG.info("Generate an operator pipeline to autogather column stats for table " + table.getTableName() + " in query " + ctx.getCmd()); ColumnStatsAutoGatherContext columnStatsAutoGatherContext = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java index d8f6898fef9e..e27fb4824b0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java @@ -19,8 +19,11 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import java.util.List; +import java.util.Set; import java.util.Locale; import java.util.Optional; import java.util.regex.Matcher; @@ -92,6 +95,17 @@ public String transformTypeString() { "[" + width + "]").orElse(""); } + /** + * The partition transforms the statement being compiled declared, or null if it declared none. + * A CREATE has to be read this way: the table it describes does not exist to be asked yet. + */ + @SuppressWarnings("unchecked") + public static List fromQueryState(Configuration conf) { + return SessionStateUtil.getResource(conf, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) + .map(spec -> (List) spec) + .orElse(null); + } + public static String toNamedStruct(List partTransformSpec, Configuration conf) { return "named_struct(" + partTransformSpec.stream().map(spec -> @@ -99,13 +113,39 @@ public static String toNamedStruct(List partTransformSpec, Config .collect(Collectors.joining(", ")) + ")"; } + + /** + * Builds the struct of source values a stats gather carries alongside each group. The transforms + * decide which rows form a partition, but only the values they were applied to let the table name + * it, and every row of a group belongs to one partition, so any of them answers for the group. + */ + public static String toSourceStruct(List partTransformSpec, Configuration conf) { + Set groupedColumns = partTransformSpec.stream() + .filter(spec -> spec.getTransformType() == TransformType.IDENTITY) + .map(TransformSpec::getColumnName) + .collect(Collectors.toSet()); + + return partTransformSpec.stream() + .map(TransformSpec::getColumnName).distinct() + .map(columnName -> { + String identifier = unparseIdentifier(columnName, conf); + // an identity transform groups by the column itself, which already answers for the group + return "'" + columnName + "', " + + (groupedColumns.contains(columnName) ? identifier : "min(" + identifier + ")"); + }) + .collect(Collectors.joining(", ", "named_struct(", ")")); + } public String toHiveExpr(Configuration conf) { - String identifier = unparseIdentifier(columnName, conf); + return toHiveExpr(unparseIdentifier(columnName, conf)); + } + + /** The transform applied to an operand, which is a column of its own table or a value of one. */ + public String toHiveExpr(String operand) { if (transformType == TransformSpec.TransformType.IDENTITY) { - return identifier; + return operand; } - String fn = "iceberg_" + transformType.name().toLowerCase() + "(" + identifier; + String fn = "iceberg_" + transformType.name().toLowerCase() + "(" + operand; switch (transformType) { case BUCKET: case TRUNCATE: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java index 53b9af9384a1..1f8fe6736b42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java @@ -464,8 +464,11 @@ private int updatePartitions(Hive db, List scs, Table table) thro } if (values.get(0).result instanceof Table) { - db.alterTable(tableFullName, (Table) values.get(0).result, environmentContext, true); - LOG.debug("Updated stats for {}.", tableFullName); + // the metastore keeps one set of counts, and they describe the table, not a branch + if (table.getSnapshotRef() == null) { + db.alterTable(tableFullName, (Table) values.get(0).result, environmentContext, true); + LOG.debug("Updated stats for {}.", tableFullName); + } } else { if (values.get(0).result instanceof Partition) { List results = Lists.transform(values, StatCollector.EXTRACT_RESULT_FUNCTION); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 55072575a08e..7c357b680409 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -160,7 +160,7 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met // and then if it is not followed by column stats, we should clean // column stats // FIXME: move this to ColStat related part - if (!work.isExplicitAnalyze() && !followedColStats1) { + if (!work.isExplicitAnalyze() && !followedColStats1 && !rewritesWithoutChangingRows()) { StatsSetupConst.clearColumnStatsState(parameters); } @@ -218,6 +218,15 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met return p.getOutput(); } + /** + * Whether this write rewrites files without changing a row, as a compaction does: what it did + * not gather still describes the table. A compaction that is meant to refresh the statistics + * recomputes them, here or in the worker that ran it. + */ + private boolean rewritesWithoutChangingRows() { + return SessionState.get() != null && SessionState.get().isCompaction(); + } + public void collectFileStatus(Warehouse wh, HiveConf conf) throws MetaException, IOException { this.conf = conf; if (providedBasicStats == null) { @@ -315,7 +324,7 @@ private int aggregateStats(Hive db, Table tbl) { } } - List partitions = getPartitionsList(db); + List partitions = getPartitionsList(db, tbl); String tableFullName = table.getDbName() + "." + table.getTableName(); @@ -328,7 +337,10 @@ private int aggregateStats(Hive db, Table tbl) { if (res == null) { return 0; } - db.alterTable(tableFullName, res, environmentContext, true); + // the metastore keeps one set of counts, and they describe the table, not a branch + if (table.getSnapshotRef() == null) { + db.alterTable(tableFullName, res, environmentContext, true); + } TransactionalStatsProcessor transactionalStatsProcessor = new TransactionalStatsProcessor(db, p); transactionalStatsProcessor.process(statsAggregator); @@ -509,17 +521,22 @@ private String toString(Map parameters) { * @return a list of partitions that need to update statistics. * @throws HiveException */ - private List getPartitionsList(Hive db) throws HiveException { + private List getPartitionsList(Hive db, Table tbl) throws HiveException { if (work.getLoadFileDesc() != null) { return null; //we are in CTAS, so we know there are no partitions } + // the handler keeps this table's partitions: return before the branches replace the task's table + if (tbl.hasNonNativePartitionSupport()) { + return null; + } + if (work.getTableSpecs() != null) { // ANALYZE command TableSpec tblSpec = work.getTableSpecs(); table = tblSpec.tableHandle; - if (!table.isPartitioned() || table.hasNonNativePartitionSupport()) { + if (!table.isPartitioned()) { return null; } // get all partitions that match with the partition spec @@ -529,7 +546,7 @@ private List getPartitionsList(Hive db) throws HiveException { // INSERT OVERWRITE command LoadTableDesc tbd = work.getLoadTableDesc(); table = db.getTable(tbd.getTable().getTableName()); - if (!table.isPartitioned() || table.hasNonNativePartitionSupport()) { + if (!table.isPartitioned()) { return null; } DynamicPartitionCtx dpCtx = tbd.getDPCtx(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 45c5dc6cdd6d..47bf5c43cb68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -21,9 +21,14 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Queue; +import java.util.NoSuchElementException; +import java.util.Iterator; import java.util.Collection; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -64,7 +69,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class ColStatsProcessor implements IStatsProcessor { private static final Logger LOG = LoggerFactory.getLogger(ColStatsProcessor.class); @@ -101,6 +105,9 @@ public int process(Hive db, Table tbl) throws Exception { private boolean constructColumnStatsFromPackedRows(Table tbl, List stats, long maxNumStats) throws HiveException, MetaException, IOException { String partName = null; + // the rows of one fetch share an inspector, so the table is asked once how to name them + Function partitionNameResolver = null; + List colName = colStatDesc.getColName(); List colType = colStatDesc.getColType(); boolean isTblLevel = colStatDesc.isTblLevel(); @@ -142,22 +149,22 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List partColSchema = new ArrayList<>(); - List partVals = new ArrayList<>(); - if (tbl.hasNonNativePartitionSupport()) { ObjectInspector inspector = fields.get(pos).getFieldObjectInspector(); - if (inspector.getCategory() == ObjectInspector.Category.STRUCT) { - Object obj = values.get(pos); - StructObjectInspector oi = (StructObjectInspector) inspector; - - for (StructField field : oi.getAllStructFieldRefs()) { - partColSchema.add(new FieldSchema(field.getFieldName(), null, "")); - partVals.add(String.valueOf(oi.getStructFieldData(obj, field))); + if (inspector instanceof StructObjectInspector oi) { + // a write groups by the partition transforms, so only the table can name the group + if (partitionNameResolver == null) { + partitionNameResolver = tbl.getStorageHandler().partitionNameResolver(tbl, oi); } + partName = partitionNameResolver.apply(values.get(pos)); + } else if (inspector instanceof PrimitiveObjectInspector poi) { + // an ANALYZE rewrite groups by the read-side partition name the reader materialized + Object partVal = poi.getPrimitiveJavaObject(values.get(pos)); + partName = partVal == null ? null : partVal.toString(); } } else { - partColSchema.addAll(tbl.getPartCols()); + List partColSchema = new ArrayList<>(tbl.getPartCols()); + List partVals = new ArrayList<>(); // Iterate over partition columns to figure out partition name for (int i = pos; i < pos + partColSchema.size(); i++) { Object partVal = ((PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector()) @@ -165,8 +172,8 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List stats = columnStatsIterator(tbl, maxNumStats); + if (stats.hasNext()) { + boolean success = tbl.getStorageHandler().setColStatistics(tbl, stats); + // COLUMN_STATS_ACCURATE describes the table, so a branch write leaves it alone + if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary()) && tbl.getSnapshotRef() == null) { + setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); + } + } + return 0; + } while (!done) { List colStats = new ArrayList<>(); @@ -237,26 +262,53 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce } start = System. currentTimeMillis(); - if (tbl.isNonNative() && tbl.getStorageHandler().canSetColStatistics(tbl)) { - boolean success = tbl.getStorageHandler().setColStatistics(tbl, colStats); - if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary())) { - setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); - } - } else { - db.setPartitionColumnStatistics(request); - } + db.setPartitionColumnStatistics(request); end = System.currentTimeMillis(); LOG.info("Time taken to update " + colStats.size() + " stats : " + ((end - start)/1000F) + " seconds."); } return 0; } + /** The computed statistics, fetched and decoded a batch at a time as they are pulled. */ + private Iterator columnStatsIterator(Table tbl, long maxNumStats) { + return new Iterator<>() { + private final Queue batch = new ArrayDeque<>(); + private boolean done = false; + + @Override + public boolean hasNext() { + while (batch.isEmpty() && !done) { + List next = new ArrayList<>(); + try { + done = constructColumnStatsFromPackedRows(tbl, next, maxNumStats); + } catch (HiveException | IOException | MetaException e) { + // a pull the interface cannot declare checked exceptions for + throw new RuntimeException(e); + } + batch.addAll(next); + } + return !batch.isEmpty(); + } + + @Override + public ColumnStatistics next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return batch.poll(); + } + }; + } + @Override public void setDpPartSpecs(Collection dpPartSpecs) { } private void setOrRemoveColumnStatsAccurateProperty(Hive db, Table tbl, List colNames, boolean success) throws HiveException { - if (CollectionUtils.isEmpty(colNames) || !colStatDesc.isTblLevel()) { + // a storage handler table has no HMS partition objects to carry per-partition flags: + // the table-level flag summarizes its partition statistics as a whole + boolean tableLevelFlag = colStatDesc.isTblLevel() || tbl.isNonNative(); + if (CollectionUtils.isEmpty(colNames) || !tableLevelFlag) { return; } EnvironmentContext environmentContext = new EnvironmentContext(); @@ -370,7 +422,6 @@ public enum ColumnStatsType { ColumnStatsField.BITVECTOR, ColumnStatsField.KLL_SKETCH)); - private final List columnStats; ColumnStatsType(List columnStats) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 1fbe66a4ecf4..e0a2b5f3d9cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -392,6 +392,36 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p } if (needColStats) { + + if (table.isNonNative() && !isPartitionStats(table, conf)) { + // the table maintains table-level column statistics only: serve them over the pruned + // set, on top of the partition-derived basic statistics + List colStats = getTableColumnStats(table, neededColumns, colStatsCache, fetchColStats); + if (estimateStats) { + colStats = estimateStatsForMissingCols(neededColumns, colStats, conf, nr, schema); + } + // we should have stats for all columns (estimated or actual) + if (neededColumns.size() == colStats.size()) { + long betterDS = getDataSizeFromColumnStats(nr, colStats); + stats.setDataSize((betterDS < 1 || colStats.isEmpty()) ? ds : betterDS); + } + // infer if any column can be primary key based on column statistics + inferAndSetPrimaryKey(stats.getNumRows(), colStats); + + stats.setColumnStatsState(deriveStatType(colStats, neededColumns)); + // they describe the whole table, so a scan pruned to part of it is described in part + if (!partList.getReferredPartCols().isEmpty() || partList.hasUnknownPartitions()) { + stats.updateColumnStatsState(State.PARTIAL); + } + stats.addToColumnStats(colStats); + + if (partStats.isEmpty()) { + // all partitions are filtered by partition pruning + stats.setBasicStatsState(State.COMPLETE); + } + return stats; + } + List partitionCols = getPartitionColumns(schema, neededColumns, referencedColumns); // We will retrieve stats from the metastore only for columns that are not cached @@ -1995,8 +2025,20 @@ public static Range combineRange(Range range1, Range range2) { } public static boolean isPartitionStats(Table table, HiveConf conf) { - return conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && table.isPartitioned() - && (!table.isNonNative() || table.getStorageHandler().canSetColStatistics(table)); + return table.isPartitioned() && isPartitionStatsEnabled(table, conf); + } + + /** + * Whether this table's statistics are kept per partition, leaving aside whether it has any. A + * CREATE has to ask this way, since the table it is about to write does not exist to be asked. + */ + public static boolean isPartitionStatsEnabled(Table table, HiveConf conf) { + // the metastore keeps a single row of column statistics per table, with nowhere to put a + // partition's, so a table partitioned outside it can only keep them per partition itself + if (table.isNonNative()) { + return table.getStorageHandler().canSetColStatistics(table, true); + } + return conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS); } public static boolean checkCanProvideStats(Table table) { @@ -2025,7 +2067,12 @@ public static boolean areBasicStatsUptoDateForQueryAnswering(Table table, Map params, String colName) { - return checkCanProvideStats(table) && StatsSetupConst.areColumnStatsUptoDate(params, colName); + // a handler keeps its own statistics and knows what happened to them, including writes by + // other engines that never touched the metastore marker + return checkCanProvideStats(table) && ( + table.isNonNative() ? table.getStorageHandler().areColumnStatsUptoDate(table, colName) : + StatsSetupConst.areColumnStatsUptoDate(params, colName) + ); } /** diff --git a/ql/src/test/queries/clientpositive/stats_part.q b/ql/src/test/queries/clientpositive/stats_part.q index d0812e100781..1b94ed673867 100644 --- a/ql/src/test/queries/clientpositive/stats_part.q +++ b/ql/src/test/queries/clientpositive/stats_part.q @@ -45,6 +45,9 @@ explain select count(key) from stats_part; --select count(*) from stats_part where p = 100; explain select count(key) from stats_part where p > 100; --select count(*) from stats_part where p > 100; +-- pruned to no partitions, max has no greatest to name +explain select max(key) from stats_part; +select max(key) from stats_part; desc formatted stats_part; --explain insert into table stats_part partition(p=100) select distinct key, value from mysource where p == 100; diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index c474362fa244..a69590cc2f21 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -134,6 +134,33 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: explain select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: explain select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +NULL PREHOOK: query: desc formatted stats_part PREHOOK: type: DESCTABLE PREHOOK: Input: default@stats_part