From 987043a2a61bd460dd7504d9a95230c48f0fbf12 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 20 Aug 2026 16:45:24 +0300 Subject: [PATCH 01/15] HIVE-29781: Introduce PARTITION__NAME virtual column and refactor Iceberg virtual column plumbing Adds VirtualColumn.PARTITION_NAME, materialized by the Iceberg readers: vectorized via HiveBatchIterator, row mode via PartitionInfo -> IOContext -> MapOperator. Refactors the read path to take everything the file already carries from the scan task instead of the per-record constants map: spec id, partition hash, file path, partition name, first row id and file sequence number are computed once per task in both VirtualColumnAwareIterator and HiveBatchIterator. Drops the now unread _partition column from the ACID read schema, along with the write-only VIRTUAL_COLS_TO_META_COLS map and the per-record helpers it fed. Fixes VectorizedParquetRecordReader taking a column type from the job column list while indexing it with a batch column number: the two orders diverge past the data columns because the batch drops virtual columns the readers cannot fill. --- .../iceberg/mr/hive/IcebergAcidUtil.java | 93 ++++++++----------- .../iceberg/mr/hive/IcebergTableUtil.java | 6 +- .../mr/hive/vector/HiveBatchIterator.java | 60 ++++++------ .../mr/hive/vector/HiveDeleteFilter.java | 2 +- .../mr/hive/vector/HiveVectorizedReader.java | 8 +- .../AbstractIcebergRecordReader.java | 2 +- .../mapreduce/IcebergMergeRecordReader.java | 5 +- .../mr/mapreduce/IcebergRecordReader.java | 24 ++--- .../positive/iceberg_explain_formatted.q.out | 6 ++ .../positive/iceberg_merge_schema.q.out | 10 +- .../llap/vectorized_iceberg_read_mixed.q.out | 14 +-- .../llap/vectorized_iceberg_read_orc.q.out | 14 +-- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../vectorized_iceberg_merge_mixed.q.out | 60 ++++++------ .../vectorized_iceberg_read_mixed.q.out | 14 +-- .../vectorized_iceberg_read_orc.q.out | 14 +-- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../hadoop/hive/ql/exec/MapOperator.java | 6 ++ .../apache/hadoop/hive/ql/io/IOContext.java | 15 ++- .../vector/VectorizedParquetRecordReader.java | 6 +- .../apache/hadoop/hive/ql/metadata/Table.java | 15 ++- .../hive/ql/metadata/VirtualColumn.java | 4 +- .../ql/optimizer/physical/Vectorizer.java | 1 + 23 files changed, 210 insertions(+), 185 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java index c615c1d82c29..0c540e6f01f1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java @@ -27,13 +27,14 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.io.PositionDeleteInfo; import org.apache.hadoop.hive.ql.io.RowLineageInfo; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.iceberg.ContentFile; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; @@ -51,16 +52,13 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.SerializationUtil; -import org.apache.iceberg.util.StructProjection; public class IcebergAcidUtil { private IcebergAcidUtil() { } - private static final Types.NestedField PARTITION_STRUCT_META_COL = null; // placeholder value in the map private static final Map FILE_READ_META_COLS = Maps.newLinkedHashMap(); - private static final Map VIRTUAL_COLS_TO_META_COLS = Maps.newLinkedHashMap(); public static final String META_TABLE_PROPERTY = "metaTable"; private static final Map DELETE_FILE_META_COLS = Maps.newLinkedHashMap(); public static final Integer PARTITION_PROJECTION_COLUMN_ID = Integer.MAX_VALUE - 6; @@ -71,17 +69,8 @@ private IcebergAcidUtil() { DELETE_FILE_META_COLS.put(MetadataColumns.ROW_POSITION, 1); FILE_READ_META_COLS.put(MetadataColumns.SPEC_ID, 0); - FILE_READ_META_COLS.put(PARTITION_STRUCT_META_COL, 1); - FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 2); - FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_SPEC_ID.getName(), MetadataColumns.SPEC_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_HASH.getName(), PARTITION_STRUCT_META_COL); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.FILE_PATH.getName(), MetadataColumns.FILE_PATH); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_POSITION.getName(), MetadataColumns.ROW_POSITION); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_LINEAGE_ID.getName(), MetadataColumns.ROW_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER.getName(), - MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER); + FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 1); + FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 2); } private static final Types.NestedField PARTITION_HASH_META_COL = Types.NestedField.required( @@ -101,18 +90,11 @@ private IcebergAcidUtil() { /** * @param dataCols The columns of the original file read schema - * @param table The table object - it is used for populating the partition struct meta column * @return The schema for reading files, extended with metadata columns */ - public static Schema createFileReadSchemaWithVirtualColums(List dataCols, Table table) { + public static Schema createFileReadSchemaWithVirtualColums(List dataCols) { List cols = Lists.newArrayListWithCapacity(dataCols.size() + FILE_READ_META_COLS.size()); - FILE_READ_META_COLS.forEach((metaCol, index) -> { - if (metaCol == PARTITION_STRUCT_META_COL) { - cols.add(MetadataColumns.metadataColumn(table, MetadataColumns.PARTITION_COLUMN_NAME)); - } else { - cols.add(metaCol); - } - }); + FILE_READ_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); cols.addAll(dataCols); return new Schema(cols); } @@ -169,12 +151,6 @@ public static int parseSpecId(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.SPEC_ID), Integer.class); } - public static long computePartitionHash(Record rec) { - StructProjection part = rec.get(FILE_READ_META_COLS.get(PARTITION_STRUCT_META_COL), StructProjection.class); - // we need to compute a hash value for the partition struct so that it can be used as a sorting key - return computeHash(part); - } - public static PartitionKey parsePartitionKey(Record rec) { String serializedStr = rec.get(SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); return SerializationUtil.deserializeFromBase64(serializedStr); @@ -190,15 +166,11 @@ public static String getSerializedPartitionKey(StructLike structLike, PartitionS return SerializationUtil.serializeToBase64(partitionKey); } - public static String parseFilePath(Record rec) { - return rec.get(FILE_READ_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - } - public static String getFilePath(Record rec) { return rec.get(DELETE_FILE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); } - public static long parseFilePosition(Record rec) { + public static long getFilePosition(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); } @@ -261,12 +233,23 @@ public static class VirtualColumnAwareIterator implements CloseableIterator currentIterator, Schema expectedSchema, Configuration conf) { + private final int specId; + private final long partitionHash; + private final String filePath; + + public VirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, + Configuration conf, FileScanTask task) { this.currentIterator = currentIterator; - this.current = GenericRecord.create( - new Schema(expectedSchema.columns().subList(4, expectedSchema.columns().size()))); + this.current = GenericRecord.create(new Schema( + expectedSchema.columns().subList(FILE_READ_META_COLS.size(), expectedSchema.columns().size()))); this.conf = conf; + + this.specId = task.file().specId(); + this.partitionHash = computeHash(task.file().partition()); + this.filePath = task.file().location(); + + IOContextMap.get(conf).setPartitionName( + IcebergTableUtil.toPartitionName(task.spec(), task.file().partition())); } @Override @@ -285,10 +268,10 @@ public T next() { GenericRecord rec = (GenericRecord) next; IcebergAcidUtil.copyFields(rec, FILE_READ_META_COLS.size(), current.size(), current); PositionDeleteInfo.setIntoConf(conf, - IcebergAcidUtil.parseSpecId(rec), - IcebergAcidUtil.computePartitionHash(rec), - IcebergAcidUtil.parseFilePath(rec), - IcebergAcidUtil.parseFilePosition(rec), + specId, + partitionHash, + filePath, + IcebergAcidUtil.getFilePosition(rec), StringUtils.EMPTY); RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec), RowLineageReader.readLastUpdatedSequenceNumber(rec), conf); @@ -300,16 +283,20 @@ public static class MergeTaskVirtualColumnAwareIterator implements CloseableI private final CloseableIterator currentIterator; private final GenericRecordBuilder recordBuilder; - private final PartitionSpec partitionSpec; - private final StructLike partition; - public MergeTaskVirtualColumnAwareIterator( - CloseableIterator currentIterator, Schema expectedSchema, ContentFile contentFile, Table table) { + private final int specId; + private final long partitionHash; + private final String serializedPartitionKey; + + public MergeTaskVirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, + PartitionSpec spec, ContentFile file) { this.currentIterator = currentIterator; - this.partition = contentFile.partition(); this.recordBuilder = new GenericRecordBuilder<>( new Schema(expectedSchema.columns().subList(0, expectedSchema.columns().size()))); - this.partitionSpec = table.specs().get(contentFile.specId()); + + this.specId = spec.specId(); + this.partitionHash = computeHash(file.partition()); + this.serializedPartitionKey = getSerializedPartitionKey(file.partition(), spec); } @Override @@ -326,11 +313,13 @@ public boolean hasNext() { public T next() { T next = currentIterator.next(); GenericRecord rec = (GenericRecord) next; - return recordBuilder.withSpecId(partitionSpec.specId()) - .withPartitionHash(computeHash(partition)) + + return recordBuilder.withSpecId(specId) + .withPartitionHash(partitionHash) .withFilePath(IcebergAcidUtil.getFilePath(rec)) .withFilePosition(IcebergAcidUtil.getDeleteFilePosition(rec)) - .withPartitionKey(getSerializedPartitionKey(partition, partitionSpec)).build(); + .withPartitionKey(serializedPartitionKey) + .build(); } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 897771bbb3cc..728c69f98f7d 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -493,8 +493,10 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy * name an unpartitioned spec renders. Statistics and partition pruning join on this name, so both must * render it the same way. */ - static String toPartitionName(PartitionSpec spec, PartitionData data) { - return StringUtils.defaultIfEmpty(spec.partitionToPath(data), DummyPartition.VOID); + public static String toPartitionName(PartitionSpec spec, StructLike data) { + String path = spec.partitionToPath(data); + // an unpartitioned spec renders nothing: its rows belong to the table-level partition + return path.isEmpty() ? DummyPartition.VOID : path; } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java index 68a910a950ae..aa2c2be78b4a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Map; import java.util.stream.LongStream; import org.apache.hadoop.hive.llap.LlapHiveUtils; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; @@ -34,10 +33,10 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.mr.hive.IcebergAcidUtil; -import org.apache.iceberg.util.StructProjection; +import org.apache.iceberg.mr.hive.IcebergTableUtil; /** * Iterator wrapper around Hive's VectorizedRowBatch producer (MRv1 implementing) record readers. @@ -52,17 +51,32 @@ public final class HiveBatchIterator implements CloseableIterator idToConstant; + + private final int specId; + private final long partitionHash; + private final String filePath; + private final String partitionName; + + private final Long firstRowId; + private final Long fileSequenceNumber; HiveBatchIterator(RecordReader recordReader, JobConf job, - int[] partitionColIndices, Object[] partitionValues, Map idToConstant) { + int[] partitionColIndices, Object[] partitionValues, FileScanTask task) { this.recordReader = recordReader; this.key = recordReader.createKey(); this.batch = recordReader.createValue(); this.vrbCtx = LlapHiveUtils.findMapWork(job).getVectorizedRowBatchCtx(); this.partitionColIndices = partitionColIndices; this.partitionValues = partitionValues; - this.idToConstant = idToConstant; + + this.specId = task.file().specId(); + this.partitionHash = IcebergAcidUtil.computeHash(task.file().partition()); + this.filePath = task.file().location(); + this.partitionName = IcebergTableUtil.toPartitionName(task.spec(), task.file().partition()); + + this.firstRowId = task.file().firstRowId(); + this.fileSequenceNumber = task.file().fileSequenceNumber(); + RowLineageUtils.initializeRowLineageColumns(vrbCtx, batch); } @@ -99,24 +113,14 @@ private void advance() { int idx = vrbCtx.findVirtualColumnNum(vc); switch (vc) { case PARTITION_SPEC_ID: - value = idToConstant.get(MetadataColumns.SPEC_ID.fieldId()); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], specId, idx); break; case PARTITION_HASH: - value = IcebergAcidUtil.computeHash( - (StructProjection) idToConstant.get(MetadataColumns.PARTITION_COLUMN_ID)); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], partitionHash, idx); break; case FILE_PATH: - value = idToConstant.get(MetadataColumns.FILE_PATH.fieldId()); BytesColumnVector bcv = (BytesColumnVector) batch.cols[idx]; - if (value == null) { - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; - } else { - bcv.fill(((String) value).getBytes()); - } + bcv.fill(filePath.getBytes()); break; case ROW_POSITION: value = LongStream.range(rowOffset, rowOffset + batch.size).toArray(); @@ -132,10 +136,13 @@ private void advance() { bcv.isNull[0] = true; bcv.isRepeating = true; break; + case PARTITION_NAME: + bcv = (BytesColumnVector) batch.cols[idx]; + bcv.fill(partitionName.getBytes()); + break; case ROW_LINEAGE_ID: LongColumnVector rowIdLcv = (LongColumnVector) batch.cols[idx]; - Object firstRowIdObj = idToConstant.get(MetadataColumns.ROW_ID.fieldId()); - if (firstRowIdObj == null) { + if (firstRowId == null) { rowIdLcv.noNulls = false; rowIdLcv.isNull[0] = true; rowIdLcv.isRepeating = true; @@ -144,13 +151,13 @@ private void advance() { // If vector[0] is still -1, the reader didn't find the column in the file. if (rowIdLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; } } else { // Lineage data was found (could be 0). Preserve it and fill only the NULL gaps. for (int i = 0; i < batch.size; i++) { if (rowIdLcv.isNull[i]) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; rowIdLcv.isNull[i] = false; } } @@ -161,8 +168,7 @@ private void advance() { case LAST_UPDATED_SEQUENCE_NUMBER: LongColumnVector lusnLcv = (LongColumnVector) batch.cols[idx]; - Object fileSeqObj = idToConstant.get(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); - if (fileSeqObj == null) { + if (fileSequenceNumber == null) { lusnLcv.noNulls = false; lusnLcv.isNull[0] = true; lusnLcv.isRepeating = true; @@ -172,13 +178,13 @@ private void advance() { // If vector[0] is still -1, apply the file-level sequence number to the whole batch. if (lusnLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; } } else { // Lineage data found in file, fill only the gaps where data is missing. for (int i = 0; i < batch.size; i++) { if (!lusnLcv.noNulls && lusnLcv.isNull[i]) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; lusnLcv.isNull[i] = false; } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java index 73ea486a3397..2444132705d0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java @@ -88,7 +88,7 @@ public CloseableIterable filterBatch(CloseableIterable iterator = new DeleteFilterBatchIterator(batches); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java index bfc7cc1fc5ba..0bbc5aa8f08e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java @@ -184,7 +184,7 @@ protected DeleteLoader newDeleteLoader() { } CloseableIterable vrbIterable = - createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, idToConstant); + createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, task); return deleteFilter != null ? deleteFilter.filterBatch(vrbIterable) : vrbIterable; @@ -293,12 +293,12 @@ private static RecordReader parquetRecordReade private static CloseableIterable createVectorizedRowBatchIterable( RecordReader hiveRecordReader, JobConf job, int[] partitionColIndices, - Object[] partitionValues, Map idToConstant) { + Object[] partitionValues, FileScanTask task) { HiveBatchIterator iterator = - new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, idToConstant); + new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, task); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java index 29f285cf37b1..a0a51b840626 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java @@ -81,7 +81,7 @@ private static Schema readSchema(Configuration conf, Table table, boolean caseSe } if (InputFormatConfig.fetchVirtualColumns(conf)) { - readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns(), table); + readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns()); if (IcebergTableUtil.supportsRowLineage(table.properties())) { readSchema = MetadataColumns.schemaWithRowLineage(readSchema); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java index 8c0dac35f1c1..a86f3954dbba 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java @@ -26,6 +26,7 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.avro.PlannedDataReader; @@ -58,8 +59,10 @@ private CloseableIterator nextTask() { CloseableIterator closeableIterator = openGeneric(mergeSplit.getContentFile(), table.schema()).iterator(); if (mergeSplit.getContentFile() instanceof DeleteFile) { Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns()); + PartitionSpec spec = table.specs().get(mergeSplit.getContentFile().specId()); + return new IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator, - deleteSchema, mergeSplit.getContentFile(), table); + deleteSchema, spec, mergeSplit.getContentFile()); } else { return closeableIterator; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 635a311a6c00..792d97c73f17 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -37,8 +37,6 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; @@ -86,12 +84,13 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { } private CloseableIterator nextTask() { - CloseableIterator closeableIterator = open(tasks.next(), expectedSchema).iterator(); + FileScanTask task = tasks.next(); + CloseableIterator closeableIterator = open(task, expectedSchema).iterator(); if (!isFetchVirtualColumns() || Utilities.getIsVectorized(conf)) { return closeableIterator; } return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator, - expectedSchema, conf); + expectedSchema, conf, task); } @Override @@ -262,19 +261,12 @@ private CloseableIterable newOrcIterable( } private Map constantsMap(FileScanTask task, BiFunction converter) { - PartitionSpec spec = task.spec(); - Set idColumns = spec.identitySourceIds(); - Schema partitionSchema = TypeUtil.select(expectedSchema, idColumns); - boolean projectsIdentityPartitionColumns = !partitionSchema.columns().isEmpty(); - if (expectedSchema.findField(MetadataColumns.PARTITION_COLUMN_ID) != null) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else if (projectsIdentityPartitionColumns) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else { - return Collections.emptyMap(); + boolean projectsIdentityPartitionColumns = !TypeUtil.select(expectedSchema, task.spec().identitySourceIds()) + .columns().isEmpty(); + if (expectedSchema.findField(MetadataColumns.SPEC_ID.fieldId()) != null || projectsIdentityPartitionColumns) { + return PartitionUtil.constantsMap(task, converter); } + return Collections.emptyMap(); } private static Schema schemaWithoutConstantsAndMeta(Schema readSchema, Map idToConstant) { diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out index 0032d06a3584..2382567e0e53 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out @@ -67,6 +67,12 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### "precision": 2147483647, "name": "PARTITION__PROJECTION" }, + { + "type": "VARCHAR", + "nullable": true, + "precision": 2147483647, + "name": "PARTITION__NAME" + }, { "type": "BIGINT", "nullable": true, diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out index 4cd3f87b9aaf..45a08b468787 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -202,7 +202,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string, 11:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -210,14 +210,14 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [6, 7, 8, 9, 10, 11, 1, 2, 3, 4, 5] - selectExpressions: ConstantVectorExpression(val 1090969) -> 11:bigint + projectedOutputColumnNums: [6, 7, 8, 9, 10, 12, 1, 2, 3, 4, 5] + selectExpressions: ConstantVectorExpression(val 1090969) -> 12:bigint Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumns: 4:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 11:bigint, 1:string, 2:string, 3:int, 5:string + valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 12:bigint, 1:string, 2:string, 3:int, 5:string Filter Vectorization: className: VectorFilterOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index 37bff7329394..a5d2bdd45ad5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -235,7 +235,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -390,7 +390,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -399,7 +399,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -516,7 +516,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -916,7 +916,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index c215c4005a46..ecc0674d0785 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -139,7 +139,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -359,7 +359,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -368,7 +368,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -520,7 +520,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -655,7 +655,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -768,7 +768,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -942,7 +942,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index b2ea3e67b861..3a458a35794e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -357,7 +357,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -558,7 +558,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 6965a3604c45..9b971e5703ff 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -338,15 +338,15 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] Select Operator expressions: ss_sold_time_sk (type: int), ss_item_sk2 (type: int), ss_customer_sk2 (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)), (floor((UDFToDouble(ss_item_sk2) / 1000.0D)) * 1000L) BETWEEN 1000L AND 2000L (type: boolean), (ss_ext_discount_amt < 0) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] - selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean + projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 33, 34] + selectExpressions: LongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 30:double)(children: DoubleColDivideDoubleScalar(col 29:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 29:double) -> 30:double) -> 31:bigint) -> 32:bigint) -> 33:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 34:boolean Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -358,16 +358,16 @@ STAGE PLANS: 0 _col2 (type: int), _col1 (type: int) 1 _col8 (type: int), _col7 (type: int) Map Join Vectorization: - bigTableFilterExpressions: SelectColumnIsTrue(col 32:boolean), SelectColumnIsTrue(col 33:boolean) + bigTableFilterExpressions: SelectColumnIsTrue(col 33:boolean), SelectColumnIsTrue(col 34:boolean) bigTableKeyColumns: 3:int, 2:int bigTableRetainColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] bigTableValueColumns: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) className: VectorMapJoinOuterMultiKeyOperator native: true nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - outerSmallTableKeyMapping: 2 -> 41, 3 -> 42 - projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) - smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + outerSmallTableKeyMapping: 2 -> 42, 3 -> 43 + projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + smallTableValueMapping: 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) hashTableImplementationType: OPTIMIZED outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 input vertices: @@ -379,13 +379,13 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [50, 20, 43, 49, 34, 22, 36, 8, 4, 11, 59, 48, 38, 7, 60, 12, 44, 18, 15, 55, 57, 21, 9, 45, 41, 51, 46, 5, 58, 6, 14, 13, 54, 53, 2, 17, 39, 3, 16, 19, 35, 10, 40, 52, 1, 37, 61, 42, 47, 56] + projectedOutputColumnNums: [51, 20, 44, 50, 35, 22, 37, 8, 4, 11, 60, 49, 39, 7, 61, 12, 45, 18, 15, 56, 58, 21, 9, 46, 42, 52, 47, 5, 59, 6, 14, 13, 55, 54, 2, 17, 40, 3, 16, 19, 36, 10, 41, 53, 1, 38, 62, 43, 48, 57] Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -394,8 +394,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [34, 35, 36, 37, 38, 30, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 62:decimal(7,2) + projectedOutputColumnNums: [35, 36, 37, 38, 39, 31, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 63, 55, 56, 57, 58, 59, 60, 61, 62] + selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val null) -> 63:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -412,7 +412,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -421,8 +421,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [31, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 63, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 63:decimal(7,2) + projectedOutputColumnNums: [32, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 64, 55, 56, 57, 58, 59, 60, 61, 62] + selectExpressions: ConstantVectorExpression(val 2451181) -> 32:int, ConstantVectorExpression(val 0) -> 64:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -431,18 +431,18 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 42:int, 65:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int + keyColumns: 43:int, 66:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 31:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 63:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + valueColumns: 32:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 64:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 39:int)) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 43:int), SelectColumnIsNull(col 40:int)) predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -451,8 +451,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 64:int + projectedOutputColumnNums: [65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 65:int Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -461,18 +461,18 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int + keyColumns: 3:int, 67:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 67:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 64:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 65:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -481,7 +481,7 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [34, 36, 35, 37] + projectedOutputColumnNums: [35, 37, 36, 38] Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -489,7 +489,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 34:int, col 35:bigint, col 36:string, col 37:bigint + keyExpressions: col 35:int, col 36:bigint, col 37:string, col 38:bigint native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -535,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] Filter Operator Filter Vectorization: className: VectorFilterOperator @@ -549,8 +549,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [23, 24, 25, 26, 27, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int + projectedOutputColumnNums: [23, 24, 25, 26, 27, 29, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 29:int Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col8 (type: int), _col7 (type: int) @@ -563,7 +563,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int - valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 29:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) Execution mode: vectorized diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index eeb9cbfe354f..e4275452b154 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -197,7 +197,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -351,7 +351,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -360,7 +360,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -476,7 +476,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -797,7 +797,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 0b7260d84206..3e4c4a1e6e0a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -102,7 +102,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -282,7 +282,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -291,7 +291,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -442,7 +442,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -576,7 +576,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -688,7 +688,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -861,7 +861,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index b4a16f6690d8..71cff814d471 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -280,7 +280,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -480,7 +480,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 221d2ac08f6f..64bd708c37ea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -686,6 +686,12 @@ public static Object[] populateVirtualColumnValues(ExecMapperContext ctx, vcValues[i] = new Text(ctx.getIoCxt().getPositionDeleteInfo().getPartitionProjection()); } break; + case PARTITION_NAME: + vcValues[i] = null; + if (ctx.getIoCxt().getPartitionName() != null) { + vcValues[i] = new Text(ctx.getIoCxt().getPartitionName()); + } + break; case FILE_PATH: vcValues[i] = null; if (ctx.getIoCxt().getPositionDeleteInfo() != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java index ad438fd1b2e2..0dd8b3afb299 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java @@ -51,12 +51,15 @@ public class IOContext { /** * supports {@link org.apache.hadoop.hive.ql.metadata.VirtualColumn#ROWID} */ - private RecordIdentifier ri; + private RecordIdentifier ri; private boolean isDeletedRecord; private PositionDeleteInfo pdi; private RowLineageInfo rowLineageInfo; - public static enum Comparison { + // the partition of the rows the reader currently serves + private String partitionName; + + public enum Comparison { GREATER, LESS, EQUAL, @@ -199,6 +202,14 @@ public PositionDeleteInfo getPositionDeleteInfo() { return pdi; } + public void setPartitionName(String partitionName) { + this.partitionName = partitionName; + } + + public String getPartitionName() { + return partitionName; + } + public void parseRowLineageInfo(JobConf cconfiguration) { this.rowLineageInfo = RowLineageInfo.parseFromConf(cconfiguration); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 03e7e5006474..236f6f3095f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -458,9 +458,11 @@ private void checkEndOfRowGroup() throws IOException { if(!colsToInclude.isEmpty()) { for (int i = 0; i < types.size(); ++i) { columnReaders[i] = - buildVectorizedParquetReader(columnTypesList.get(colsToInclude.get(i)), types.get(i), + buildVectorizedParquetReader( + columnTypesList.get(colsToInclude.get(i)), types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, 0, 0); + legacyConversionEnabled, 0, 0 + ); } } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 2592113614cc..a46e8c2cb95a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -1463,16 +1463,21 @@ public List getVirtualColumns() { List virtualColumns = new ArrayList<>(); if (!isNonNative()) { virtualColumns.addAll(VirtualColumn.getRegistry()); + return virtualColumns; } - if (isNonNative() && AcidUtils.isNonNativeAcidTable(this)) { + if (AcidUtils.isNonNativeAcidTable(this)) { virtualColumns.addAll(getStorageHandler().acidVirtualColumns()); } - if (isNonNative() && getStorageHandler().areSnapshotsSupported() && - isBlank(getMetaTable())) { + if (!isBlank(getMetaTable())) { + return virtualColumns; + } + if (hasNonNativePartitionSupport()) { + virtualColumns.add(VirtualColumn.PARTITION_NAME); + } + if (getStorageHandler().areSnapshotsSupported()) { virtualColumns.add(VirtualColumn.SNAPSHOT_ID); } - if (isNonNative() && getStorageHandler().supportsRowLineage(getTTable().getParameters()) && - isBlank(getMetaTable())) { + if (getStorageHandler().supportsRowLineage(getTTable().getParameters())) { virtualColumns.add(VirtualColumn.ROW_LINEAGE_ID); virtualColumns.add(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index 227dda27d9d2..fb5852cf144c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -58,6 +58,7 @@ public enum VirtualColumn { ROW_POSITION("ROW__POSITION", TypeInfoFactory.longTypeInfo), SNAPSHOT_ID("SNAPSHOT__ID", TypeInfoFactory.longTypeInfo), PARTITION_PROJECTION("PARTITION__PROJECTION", TypeInfoFactory.stringTypeInfo), + PARTITION_NAME("PARTITION__NAME", TypeInfoFactory.stringTypeInfo), ROW_LINEAGE_ID("ROW__LINEAGE__ID", TypeInfoFactory.longTypeInfo), LAST_UPDATED_SEQUENCE_NUMBER("LAST__UPDATED__SEQUENCE__NUMBER", TypeInfoFactory.longTypeInfo), @@ -74,7 +75,8 @@ public enum VirtualColumn { ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName(), ROWISDELETED.getName(), PARTITION_SPEC_ID.getName(), PARTITION_HASH.getName(), FILE_PATH.getName(), ROW_POSITION.getName(), - PARTITION_PROJECTION.getName(), ROW_LINEAGE_ID.getName(), LAST_UPDATED_SEQUENCE_NUMBER.getName()); + PARTITION_PROJECTION.getName(), PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), + LAST_UPDATED_SEQUENCE_NUMBER.getName()); public static final ImmutableMap VIRTUAL_COLUMN_NAME_MAP = new ImmutableMap.Builder().putAll(getColumnNameMap()).build(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 512337c11fdb..7d0a3cb2a8cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -310,6 +310,7 @@ public class Vectorizer implements PhysicalPlanResolver { VirtualColumn.FILE_PATH, VirtualColumn.ROW_POSITION, VirtualColumn.PARTITION_PROJECTION, + VirtualColumn.PARTITION_NAME, VirtualColumn.ROW_LINEAGE_ID, VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); private HiveConf hiveConf; From ced7038c5cf1a03818b683f847a346d8f4617930 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 20 Aug 2026 19:40:03 +0300 Subject: [PATCH 02/15] HIVE-29781: Make PARTITION__PROJECTION a merge task record field instead of a virtual column The partition key a merge task carries is read back only by HiveIcebergDeleteWriter under isMergeTask: an ordinary delete derives the key from the row data, and a merge task cannot because it reads delete files, whose records hold only a path and a position. Declaring it as a virtual column put an always empty string into every DELETE, UPDATE and MERGE record and into every Iceberg plan. It now lives in a merge specific serde layout that IcebergMergeRecordReader and HiveIcebergSerDe select on the merge task flag, so the ordinary delete record is one column shorter and the plans no longer estimate it. --- .../iceberg/mr/hive/HiveIcebergSerDe.java | 17 +- .../mr/hive/HiveIcebergStorageHandler.java | 7 +- .../iceberg/mr/hive/IcebergAcidUtil.java | 72 +-- .../mr/hive/vector/HiveBatchIterator.java | 6 - .../HiveIcebergCopyOnWriteRecordWriter.java | 2 +- .../hive/writer/HiveIcebergDeleteWriter.java | 2 +- .../mapreduce/IcebergMergeRecordReader.java | 2 +- .../mr/mapreduce/IcebergRecordReader.java | 2 +- .../mr/hive/TestHiveIcebergSelects.java | 17 + .../vector/TestHiveIcebergVectorization.java | 10 +- .../writer/TestHiveIcebergDeleteWriter.java | 5 +- ...te_iceberg_copy_on_write_partitioned.q.out | 286 +++++------ ..._iceberg_copy_on_write_unpartitioned.q.out | 278 +++++------ .../positive/delete_iceberg_mixed.q.out | 10 +- .../positive/iceberg_explain_formatted.q.out | 6 - .../positive/iceberg_merge_schema.q.out | 10 +- ...rg_truncate_partition_with_evolution.q.out | 40 +- .../llap/vectorized_iceberg_read_mixed.q.out | 14 +- .../llap/vectorized_iceberg_read_orc.q.out | 14 +- .../vectorized_iceberg_read_parquet.q.out | 8 +- ...ge_iceberg_copy_on_write_partitioned.q.out | 164 +++---- ..._iceberg_copy_on_write_unpartitioned.q.out | 322 ++++++------- .../results/positive/merge_iceberg_orc.q.out | 74 +-- .../merge_iceberg_partitioned_orc.q.out | 74 +-- ...merge_with_null_check_on_joining_col.q.out | 140 +++--- .../results/positive/metadata_delete.q.out | 8 +- .../results/positive/mv_iceberg_orc4.q.out | 42 +- .../results/positive/mv_iceberg_orc5.q.out | 42 +- .../results/positive/mv_iceberg_orc7.q.out | 22 +- .../results/positive/mv_iceberg_orc8.q.out | 42 +- ...te_iceberg_copy_on_write_partitioned.q.out | 452 +++++++++--------- ..._iceberg_copy_on_write_unpartitioned.q.out | 360 +++++++------- .../vectorized_iceberg_merge_mixed.q.out | 126 ++--- .../vectorized_iceberg_read_mixed.q.out | 14 +- .../vectorized_iceberg_read_orc.q.out | 14 +- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../positive/write_iceberg_branch.q.out | 60 +-- .../hadoop/hive/ql/exec/MapOperator.java | 6 - .../hadoop/hive/ql/io/PositionDeleteInfo.java | 16 +- .../hive/ql/metadata/VirtualColumn.java | 3 +- .../ql/optimizer/physical/Vectorizer.java | 1 - 41 files changed, 1400 insertions(+), 1398 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java index 83d00942edd5..68dc85413742 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java @@ -140,8 +140,7 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - this.projectedSchema = - projectedSchema(conf, serDeProperties.getProperty(Catalogs.NAME), tableSchema, jobConf); + this.projectedSchema = projectedSchema(conf, serDeProperties, tableSchema, jobConf); if (!IcebergTableUtil.isFanoutEnabled(serDeProperties::getProperty)) { // ClusteredWriter requires that records are ordered by partition keys. @@ -156,9 +155,11 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - private static Schema projectedSchema(Configuration conf, String tableName, Schema tableSchema, - Map jobConf) { + private static Schema projectedSchema(Configuration conf, Properties serDeProperties, + Schema tableSchema, Map jobConf) { + String tableName = serDeProperties.getProperty(Catalogs.NAME); Context.Operation operation = HiveCustomStorageHandlerUtils.getWriteOperation(conf::get, tableName); + if (operation == null) { jobConf.put(InputFormatConfig.CASE_SENSITIVE, "false"); String[] selectedColumns = ColumnProjectionUtils.getReadColumnNames(conf); @@ -180,11 +181,15 @@ private static Schema projectedSchema(Configuration conf, String tableName, Sche } boolean isCOW = IcebergTableUtil.isCopyOnWriteMode(operation, conf::get); if (isCOW) { - return getSchemaWithRowLineage(IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()), conf); + return getSchemaWithRowLineage( + IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), false), conf); } switch (operation) { case DELETE: - return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()); + boolean isMergeTask = HiveCustomStorageHandlerUtils.isMergeTaskEnabled( + key -> serDeProperties.getProperty(key, conf.get(key)), + tableName); + return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), isMergeTask); case UPDATE: return IcebergAcidUtil.createSerdeSchemaForUpdate(tableSchema.columns()); case OTHER: diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b23c20991856..d79275668a67 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -214,7 +214,7 @@ import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.FILE_PATH; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_HASH; -import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_PROJECTION; +import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_NAME; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_SPEC_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_LINEAGE_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_POSITION; @@ -244,7 +244,7 @@ public class HiveIcebergStorageHandler extends DefaultStorageHandler implements public static final String MERGE_ON_READ = RowLevelOperationMode.MERGE_ON_READ.modeName(); private static final List ACID_VIRTUAL_COLS = ImmutableList.of( - PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION, PARTITION_PROJECTION); + PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION); private static final List ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA = schema(ACID_VIRTUAL_COLS); @@ -427,7 +427,8 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese List subExprNodes = pushedPredicate.getChildren(); Set skipList = - Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) + Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, PARTITION_NAME, + ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) .map(VirtualColumn::getName).collect(Collectors.toSet()); if (subExprNodes.removeIf(nodeDesc -> nodeDesc.getCols() != null && diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java index 0c540e6f01f1..b881e07b2eb8 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java @@ -25,7 +25,6 @@ import java.util.Objects; import java.util.Optional; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.io.PositionDeleteInfo; @@ -76,16 +75,22 @@ private IcebergAcidUtil() { private static final Types.NestedField PARTITION_HASH_META_COL = Types.NestedField.required( MetadataColumns.PARTITION_COLUMN_ID, MetadataColumns.PARTITION_COLUMN_NAME, Types.LongType.get()); - public static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( + private static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( PARTITION_PROJECTION_COLUMN_ID, PARTITION_PROJECTION_COLUMN_NAME, Types.StringType.get()); + private static final Map SERDE_META_COLS = Maps.newLinkedHashMap(); + // a merge task reads delete files, so its writer has no row data to derive the partition key from + private static final Map MERGE_SERDE_META_COLS = Maps.newLinkedHashMap(); + static { SERDE_META_COLS.put(MetadataColumns.SPEC_ID, 0); SERDE_META_COLS.put(PARTITION_HASH_META_COL, 1); SERDE_META_COLS.put(MetadataColumns.FILE_PATH, 2); SERDE_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - SERDE_META_COLS.put(PARTITION_PROJECTION, 4); + + MERGE_SERDE_META_COLS.putAll(SERDE_META_COLS); + MERGE_SERDE_META_COLS.put(PARTITION_PROJECTION, 4); } /** @@ -101,11 +106,14 @@ public static Schema createFileReadSchemaWithVirtualColums(List dataCols) { - List cols = Lists.newArrayListWithCapacity(dataCols.size() + SERDE_META_COLS.size()); - SERDE_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); + public static Schema createSerdeSchemaForDelete(List dataCols, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; + List cols = Lists.newArrayListWithCapacity(dataCols.size() + metaCols.size()); + cols.addAll(metaCols.keySet()); cols.addAll(dataCols); return new Schema(cols); } @@ -115,14 +123,17 @@ public static Schema createSerdeSchemaForDelete(List dataCols * the field values from `rec`. * @param rec The record read by the file scan task, which contains both the metadata fields and the row data fields * @param rowData The record object to populate with the rowData fields only + * @param isMergeTask Whether the record was built by a merge task * @return The position delete object */ - public static PositionDelete getPositionDelete(Record rec, Record rowData) { + public static PositionDelete getPositionDelete(Record rec, Record rowData, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; PositionDelete positionDelete = PositionDelete.create(); - String filePath = rec.get(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - Long filePosition = rec.get(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); + String filePath = rec.get(metaCols.get(MetadataColumns.FILE_PATH), String.class); + Long filePosition = rec.get(metaCols.get(MetadataColumns.ROW_POSITION), Long.class); - int dataOffset = SERDE_META_COLS.size(); // position in the rec where the actual row data begins + int dataOffset = metaCols.size(); // position in the rec where the actual row data begins for (int i = dataOffset; i < rec.size(); ++i) { rowData.set(i - dataOffset, rec.get(i)); } @@ -152,7 +163,7 @@ public static int parseSpecId(Record rec) { } public static PartitionKey parsePartitionKey(Record rec) { - String serializedStr = rec.get(SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); + String serializedStr = rec.get(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); return SerializationUtil.deserializeFromBase64(serializedStr); } @@ -237,11 +248,11 @@ public static class VirtualColumnAwareIterator implements CloseableIterator currentIterator, Schema expectedSchema, + public VirtualColumnAwareIterator(CloseableIterator currentIterator, List columns, Configuration conf, FileScanTask task) { this.currentIterator = currentIterator; - this.current = GenericRecord.create(new Schema( - expectedSchema.columns().subList(FILE_READ_META_COLS.size(), expectedSchema.columns().size()))); + this.current = GenericRecord.create( + new Schema(columns.subList(FILE_READ_META_COLS.size(), columns.size()))); this.conf = conf; this.specId = task.file().specId(); @@ -271,8 +282,7 @@ public T next() { specId, partitionHash, filePath, - IcebergAcidUtil.getFilePosition(rec), - StringUtils.EMPTY); + IcebergAcidUtil.getFilePosition(rec)); RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec), RowLineageReader.readLastUpdatedSequenceNumber(rec), conf); return (T) current; @@ -282,7 +292,7 @@ public T next() { public static class MergeTaskVirtualColumnAwareIterator implements CloseableIterator { private final CloseableIterator currentIterator; - private final GenericRecordBuilder recordBuilder; + private final MergeTaskRecordBuilder recordBuilder; private final int specId; private final long partitionHash; @@ -291,8 +301,7 @@ public static class MergeTaskVirtualColumnAwareIterator implements CloseableI public MergeTaskVirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, PartitionSpec spec, ContentFile file) { this.currentIterator = currentIterator; - this.recordBuilder = new GenericRecordBuilder<>( - new Schema(expectedSchema.columns().subList(0, expectedSchema.columns().size()))); + this.recordBuilder = new MergeTaskRecordBuilder<>(expectedSchema); this.specId = spec.specId(); this.partitionHash = computeHash(file.partition()); @@ -323,36 +332,35 @@ public T next() { } } - private static final class GenericRecordBuilder { - + private static final class MergeTaskRecordBuilder { private final GenericRecord current; - GenericRecordBuilder(Schema schema) { + MergeTaskRecordBuilder(Schema schema) { current = GenericRecord.create(schema); } - public GenericRecordBuilder withSpecId(int specId) { - current.set(SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); + public MergeTaskRecordBuilder withSpecId(int specId) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); return this; } - public GenericRecordBuilder withPartitionHash(long partitionHash) { - current.set(SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); + public MergeTaskRecordBuilder withPartitionHash(long partitionHash) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); return this; } - public GenericRecordBuilder withFilePath(String filePath) { - current.set(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); + public MergeTaskRecordBuilder withFilePath(String filePath) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); return this; } - public GenericRecordBuilder withFilePosition(long filePosition) { - current.set(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); + public MergeTaskRecordBuilder withFilePosition(long filePosition) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); return this; } - public GenericRecordBuilder withPartitionKey(String serializedPartitionKey) { - current.set(SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); + public MergeTaskRecordBuilder withPartitionKey(String serializedPartitionKey) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); return this; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java index aa2c2be78b4a..ef073f07a458 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java @@ -130,12 +130,6 @@ private void advance() { lcv.isRepeating = false; System.arraycopy(value, 0, lcv.vector, 0, batch.size); break; - case PARTITION_PROJECTION: - bcv = (BytesColumnVector) batch.cols[idx]; - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; - break; case PARTITION_NAME: bcv = (BytesColumnVector) batch.cols[idx]; bcv.fill(partitionName.getBytes()); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java index 4fe3b533755a..cdb03e1a89c5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java @@ -54,7 +54,7 @@ class HiveIcebergCopyOnWriteRecordWriter extends HiveIcebergDefaultWriter { @Override public void write(Writable row) throws IOException { Record record = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate, false); Record rowData = positionDelete.row(); if (positionDelete.pos() < 0) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java index 9365f9834ab7..6970556fc46a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java @@ -58,7 +58,7 @@ class HiveIcebergDeleteWriter extends HiveIcebergWriterBase { @Override public void write(Writable row) throws IOException { Record rec = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate, isMergeTask); int specId = IcebergAcidUtil.parseSpecId(rec); PartitionKey partitionKey = isMergeTask ? IcebergAcidUtil.parsePartitionKey(rec) : partition(positionDelete.row(), specId); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java index a86f3954dbba..824acf20e33f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java @@ -58,7 +58,7 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { private CloseableIterator nextTask() { CloseableIterator closeableIterator = openGeneric(mergeSplit.getContentFile(), table.schema()).iterator(); if (mergeSplit.getContentFile() instanceof DeleteFile) { - Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns()); + Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns(), true); PartitionSpec spec = table.specs().get(mergeSplit.getContentFile().specId()); return new IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator, diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 792d97c73f17..14430fdc3af2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -90,7 +90,7 @@ private CloseableIterator nextTask() { return closeableIterator; } return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator, - expectedSchema, conf, task); + expectedSchema.columns(), conf, task); } @Override diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java index f82a2f449ade..b1746c21b38e 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; @@ -285,4 +286,20 @@ public void testHistory() throws IOException, InterruptedException { Assert.assertEquals(table.history().get(i).snapshotId(), history.get(i)[0]); } } + + @Test + public void testFilterOnPartitionName() { + // PARTITION__NAME is materialized by the reader and is not a field of the Iceberg schema, so a + // predicate on it must not be pushed down to Iceberg + TableIdentifier identifier = TableIdentifier.of("default", "part_name_filter"); + testTables.createTable(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).identity("last_name").build(), + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); + + List rows = shell.executeStatement( + "SELECT customer_id FROM " + identifier.name() + + " WHERE PARTITION__NAME = 'last_name=Brown' AND customer_id = 0"); + Assert.assertEquals(1, rows.size()); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java index 2a8b0bdc6ed1..28487b715ff4 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java @@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -109,9 +110,10 @@ public void testRowIterator() throws Exception { List records = TestHelper.generateRandomRecords(allSchema, 10, 0L); Table table = testTables.createTable(shell, "temptable", allSchema, fileFormat, records); - // Identify data file location - expected to be 1 file exactly - Path dataFilePath = new Path(Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) - .files().iterator()).get(0).file().path().toString()); + // Identify the scan task - expected to be 1 file exactly + FileScanTask fileScanTask = Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) + .files().iterator()).get(0); + Path dataFilePath = new Path(fileScanTask.file().path().toString()); // Generate a mock vectorized read job JobConf jobConf = prepareMockJob(allSchema, dataFilePath); @@ -122,7 +124,7 @@ public void testRowIterator() throws Exception { inputFormat.getRecordReader(new FileSplit(dataFilePath, 0L, Long.MAX_VALUE, new String[]{}), jobConf, new MockReporter()); HiveBatchIterator hiveBatchIterator = new HiveBatchIterator( - internalVectorizedRecordReader, jobConf, null, null, null); + internalVectorizedRecordReader, jobConf, null, null, fileScanTask); // Expected to be one batch exactly HiveBatchContext hiveBatchContext = hiveBatchIterator.next(); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java index 36ff67cb2068..1f5e8cfa7a04 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java @@ -88,7 +88,8 @@ private static List deleteRecords(Table table, Set idsTo continue; } - GenericRecord deleteRecord = GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns())); + GenericRecord deleteRecord = + GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns(), false)); int specId = (Integer) record.getField(MetadataColumns.SPEC_ID.name()); deleteRecord.setField(MetadataColumns.SPEC_ID.name(), specId); PartitionKey partitionKey = new PartitionKey(table.specs().get(specId), table.schema()); @@ -96,8 +97,6 @@ private static List deleteRecords(Table table, Set idsTo deleteRecord.setField(MetadataColumns.PARTITION_COLUMN_NAME, partitionKey); deleteRecord.setField(MetadataColumns.FILE_PATH.name(), record.getField(MetadataColumns.FILE_PATH.name())); deleteRecord.setField(MetadataColumns.ROW_POSITION.name(), record.getField(MetadataColumns.ROW_POSITION.name())); - deleteRecord.setField(IcebergAcidUtil.PARTITION_PROJECTION.name(), - IcebergAcidUtil.getSerializedPartitionKey(partitionKey, table.spec())); SCHEMA.columns().forEach(field -> deleteRecord.setField(field.name(), record.getField(field.name()))); deleteRecords.add(deleteRecord); diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index a531c87a685c..97f2fb1f0303 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -126,16 +126,16 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -146,14 +146,14 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL @@ -358,15 +358,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -375,13 +375,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -390,15 +390,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -407,25 +407,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 18 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -444,21 +444,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 19 Execution mode: vectorized Reduce Operator Tree: @@ -485,13 +485,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -618,19 +618,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -639,13 +639,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -654,26 +654,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -682,29 +682,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -839,8 +839,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -848,13 +848,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -862,7 +862,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1010,13 +1010,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1025,18 +1025,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1044,7 +1044,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1053,25 +1053,25 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1089,7 +1089,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1097,13 +1097,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1128,16 +1128,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1303,8 +1303,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1312,13 +1312,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1326,7 +1326,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1474,13 +1474,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1489,18 +1489,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1508,7 +1508,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1517,25 +1517,25 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1553,7 +1553,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1561,13 +1561,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1592,16 +1592,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Reduce Operator Tree: Merge Join Operator diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out index 9a5350c2e0f2..0c539335f43f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out @@ -54,16 +54,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -92,15 +92,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -156,9 +156,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -177,17 +177,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -301,16 +301,16 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -321,14 +321,14 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (a <= 5) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE @@ -511,15 +511,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -528,13 +528,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -543,15 +543,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -560,25 +560,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -597,17 +597,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1431 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 879 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1455 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -667,13 +667,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -772,19 +772,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -793,13 +793,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -808,26 +808,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -836,15 +836,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -993,8 +993,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1002,13 +1002,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1016,7 +1016,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1151,13 +1151,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1166,18 +1166,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1185,7 +1185,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1194,11 +1194,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1216,7 +1216,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1224,13 +1224,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1255,8 +1255,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1431,8 +1431,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1440,13 +1440,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1454,7 +1454,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1583,13 +1583,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1598,18 +1598,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1617,7 +1617,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1626,11 +1626,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1648,7 +1648,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1656,13 +1656,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1687,8 +1687,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out index aed511e4b8fe..d8f7ecf10e8d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out @@ -85,14 +85,14 @@ Stage-4 File Output Operator [FS_46] table:{"name:":"default.ice01"} Select Operator [SEL_44] (rows=7 width=206) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_43] (rows=7 width=206) - Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_59] PartitionCols:_col4 Select Operator [SEL_56] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_53] (rows=7 width=188) predicate:((((id = 2) or (id > 4)) is null or (id < 2) or ((id > 2) and (id <= 4))) and FILE__PATH is not null) TableScan [TS_0] (rows=7 width=188) @@ -120,13 +120,13 @@ Stage-4 File Output Operator [FS_70] table:{"name:":"default.ice01"} Select Operator [SEL_69] (rows=3 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_68] (rows=3 width=188) predicate:(row_number_window_0 = 1) PTF Operator [PTF_67] (rows=7 width=188) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] Select Operator [SEL_66] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_58] PartitionCols:FILE__PATH diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out index 2382567e0e53..f6b8cfea653d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out @@ -61,12 +61,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### "nullable": true, "name": "ROW__POSITION" }, - { - "type": "VARCHAR", - "nullable": true, - "precision": 2147483647, - "name": "PARTITION__PROJECTION" - }, { "type": "VARCHAR", "nullable": true, diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out index 45a08b468787..4eac820ba403 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -202,7 +202,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string, 11:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -210,14 +210,14 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [6, 7, 8, 9, 10, 12, 1, 2, 3, 4, 5] - selectExpressions: ConstantVectorExpression(val 1090969) -> 12:bigint + projectedOutputColumnNums: [6, 7, 8, 9, 11, 1, 2, 3, 4, 5] + selectExpressions: ConstantVectorExpression(val 1090969) -> 11:bigint Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumns: 4:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 12:bigint, 1:string, 2:string, 3:int, 5:string + valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 11:bigint, 1:string, 2:string, 3:int, 5:string Filter Vectorization: className: VectorFilterOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index ea652fe65e98..d6a09b1d53c6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -90,12 +90,12 @@ STAGE PLANS: predicate: (a = 22) (type: boolean) Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -285,12 +285,12 @@ STAGE PLANS: predicate: (a = 226784902765739L) (type: boolean) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 226784902765739L (type: bigint), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 226784902765739L (type: bigint), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -506,12 +506,12 @@ STAGE PLANS: predicate: (b = 'ddd') (type: boolean) Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), 'ddd' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), 'ddd' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -747,8 +747,8 @@ STAGE PLANS: predicate: (b = DATE'2022-02-07') (type: boolean) Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -935,12 +935,12 @@ STAGE PLANS: predicate: (a = 1.156748927566759E11D) (type: boolean) Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), b (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), b (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1117,8 +1117,8 @@ STAGE PLANS: predicate: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index a5d2bdd45ad5..51f0da923f1f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -235,7 +235,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -390,7 +390,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -399,7 +399,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -516,7 +516,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -916,7 +916,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index ecc0674d0785..e2669a45789d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -139,7 +139,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -359,7 +359,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -368,7 +368,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -520,7 +520,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -655,7 +655,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -768,7 +768,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -942,7 +942,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index 3a458a35794e..0b66a1287bca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -357,7 +357,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -558,7 +558,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index b6e253358ee6..9ed2e47d7837 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -137,16 +137,16 @@ STAGE PLANS: predicate: (a <= 100) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Execution mode: vectorized Map 12 Map Operator Tree: @@ -157,30 +157,30 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL @@ -200,9 +200,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -221,21 +221,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -243,30 +243,30 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 972 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 604 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -278,24 +278,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col5 <> _col8) or _col5 is null or _col8 is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col4 <> _col7) or _col4 is null or _col7 is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -304,15 +304,15 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -382,16 +382,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -472,16 +472,16 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -506,30 +506,30 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: _col5 is null (type: boolean) + predicate: _col4 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), _col6 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out index 115992d60112..bb4c64a103ba 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out @@ -148,41 +148,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a <= 100) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -201,24 +201,24 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Execution mode: vectorized Reducer 10 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -237,17 +237,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -259,20 +259,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 2440 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 14 Data size: 1520 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1952 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1216 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new') (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3424 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new') (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2688 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -285,16 +285,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 486 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -306,24 +306,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 is null or (_col5 = _col8)) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 is null or (_col4 = _col7)) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -332,11 +332,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -411,16 +411,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -483,44 +483,44 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a > 100) (type: boolean) Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: ((a > 100) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE @@ -591,20 +591,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 14 Data size: 2420 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 14 Data size: 1500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new 2') (type: string), null (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3412 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new 2') (type: string), null (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -616,24 +616,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 <= 100) or (_col4 is null or ((_col4 = _col7) and (_col4 > 100))) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -642,11 +642,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -659,23 +659,23 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -694,17 +694,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -865,41 +865,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) + Map-reduce partition columns: _col3 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -959,16 +959,16 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col5 is null or (_col5 <> _col8) or _col8 is null) (type: boolean) + predicate: (_col4 is null or (_col4 <> _col7) or _col7 is null) (type: boolean) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -976,7 +976,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -985,7 +985,7 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1001,9 +1001,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col4 (type: int) + 0 _col3 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -1011,13 +1011,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1042,8 +1042,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1059,13 +1059,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), 'Merged' (type: string), (_col6 - 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), 'Merged' (type: string), (_col5 - 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out index dcb3ecd97145..830129a81e73 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out @@ -99,16 +99,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -117,48 +117,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -170,10 +170,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index 693513fa2dfc..c2529066adac 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -101,16 +101,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -119,48 +119,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -171,10 +171,10 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: PARTIAL + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out index 64bbd1b55dec..ca720205bf24 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out @@ -71,31 +71,31 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[+($6, 100)], t__c=[$7]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[+($5, 100)], t__c=[$6]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), IS NULL($5)), >($5, 10), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), IS NOT NULL($5)), <=($5, 10), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), IS NULL($4)), >($4, 10), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), IS NOT NULL($4)), <=($4, 10), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -111,11 +111,11 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[>($0, 10)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -141,23 +141,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(IS NULL($5), AND(=($5, $8), =($6, $9), >($5, 10)), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), IS NOT NULL($5)))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), IS NOT NULL($4)))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -174,12 +174,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(IS NULL($3), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveTableScan(table=[[default, source]], table:alias=[s]) @@ -203,21 +203,21 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[null:INTEGER], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6), IS NULL($7))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[null:INTEGER], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5), IS NULL($6))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -234,12 +234,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) @@ -264,23 +264,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -297,12 +297,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) diff --git a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out index bfa7597c38e0..d37561e23c54 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out @@ -373,12 +373,12 @@ STAGE PLANS: predicate: (b < 5) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out index f6adb53ea76c..5fbd6803a7c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out @@ -410,16 +410,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string) + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -474,25 +474,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1524 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint) + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -520,9 +520,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Filter Operator predicate: _col3 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -552,12 +552,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out index 7fbe53177787..dabdc0989f39 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out @@ -153,16 +153,16 @@ STAGE PLANS: alias: default.mat2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), _c4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), _c4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col10 (type: double) + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: double) Execution mode: vectorized Map 6 Map Operator Tree: @@ -217,25 +217,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 2172 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 6 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col10 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col9 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: double) + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -263,9 +263,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1448 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -295,12 +295,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint), VALUE._col8 (type: bigint), VALUE._col9 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint), VALUE._col7 (type: bigint), VALUE._col8 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out index 94381e415353..a52e9ae8390f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out @@ -120,8 +120,8 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -129,7 +129,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -165,25 +165,25 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) nullSafes: [true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: bigint) Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -198,7 +198,7 @@ STAGE PLANS: predicate: _col2 is null (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -213,8 +213,8 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out index 22ccaad5e4da..23948e8892ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out @@ -87,16 +87,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE + expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) null sort order: zzz sort order: +++ Map-reduce partition columns: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -135,25 +135,25 @@ STAGE PLANS: 0 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) 1 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) nullSafes: [true, true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: varchar(256)), _col6 (type: char(100)), _col7 (type: int), _col8 (type: bigint) + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: varchar(256)), _col5 (type: char(100)), _col6 (type: int), _col7 (type: bigint) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -181,9 +181,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -213,12 +213,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: varchar(256)), VALUE._col6 (type: char(100)), VALUE._col7 (type: int), VALUE._col8 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: varchar(256)), VALUE._col5 (type: char(100)), VALUE._col6 (type: int), VALUE._col7 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 7dac902dc9ae..6f3cfd8cadb7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -56,16 +56,16 @@ STAGE PLANS: predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1461 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 909 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Map 4 Map Operator Tree: @@ -77,16 +77,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -103,7 +103,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL @@ -118,12 +118,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -137,26 +137,26 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -175,21 +175,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: @@ -394,28 +394,28 @@ STAGE PLANS: alias: tbl_ice Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -426,14 +426,14 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,15 +497,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -514,22 +514,22 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -538,13 +538,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -553,19 +553,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -574,13 +574,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -589,26 +589,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -617,13 +617,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -632,15 +632,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -649,18 +649,18 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -705,9 +705,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -726,21 +726,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 21 Execution mode: vectorized Reduce Operator Tree: @@ -789,15 +789,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 24 Reduce Operator Tree: Merge Join Operator @@ -806,15 +806,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 25 Execution mode: vectorized Reduce Operator Tree: @@ -1009,29 +1009,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1177,8 +1177,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1186,13 +1186,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1200,10 +1200,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1211,7 +1211,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1320,7 +1320,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1328,13 +1328,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1359,16 +1359,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1440,25 +1440,25 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1495,13 +1495,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -1510,18 +1510,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1529,7 +1529,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1538,19 +1538,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 @@ -1688,8 +1688,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1697,13 +1697,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1711,10 +1711,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1722,7 +1722,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1831,7 +1831,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1839,13 +1839,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1870,16 +1870,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1951,25 +1951,25 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2006,13 +2006,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2021,18 +2021,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -2040,7 +2040,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -2049,19 +2049,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out index 150fa60ce166..01c7c92df472 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out @@ -55,12 +55,12 @@ STAGE PLANS: predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1948 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -96,24 +96,24 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -132,17 +132,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -202,15 +202,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -329,28 +329,28 @@ STAGE PLANS: alias: tbl_ice Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -361,14 +361,14 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,26 +497,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -525,15 +525,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -661,15 +661,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -678,13 +678,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -693,15 +693,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 20 Reduce Operator Tree: Merge Join Operator @@ -710,15 +710,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 21 Reduce Operator Tree: Merge Join Operator @@ -727,18 +727,18 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4437 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2781 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -920,13 +920,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -935,15 +935,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -952,25 +952,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -989,17 +989,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1172 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1013,13 +1013,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1028,19 +1028,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Union 7 Vertex: Union 7 @@ -1160,8 +1160,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1169,13 +1169,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1183,10 +1183,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1194,7 +1194,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1342,13 +1342,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1357,18 +1357,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1376,7 +1376,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1385,11 +1385,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1407,11 +1407,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1429,7 +1429,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1437,13 +1437,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1468,8 +1468,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1666,8 +1666,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1675,13 +1675,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1689,10 +1689,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1700,7 +1700,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1796,7 +1796,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1804,13 +1804,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1835,8 +1835,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1917,11 +1917,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1958,13 +1958,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1973,18 +1973,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1992,7 +1992,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2001,11 +2001,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 9b971e5703ff..bcb446bbfdb5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -338,15 +338,15 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Select Operator expressions: ss_sold_time_sk (type: int), ss_item_sk2 (type: int), ss_customer_sk2 (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)), (floor((UDFToDouble(ss_item_sk2) / 1000.0D)) * 1000L) BETWEEN 1000L AND 2000L (type: boolean), (ss_ext_discount_amt < 0) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 33, 34] - selectExpressions: LongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 30:double)(children: DoubleColDivideDoubleScalar(col 29:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 29:double) -> 30:double) -> 31:bigint) -> 32:bigint) -> 33:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 34:boolean + projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] + selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -355,47 +355,47 @@ STAGE PLANS: 0 {_col22} {_col23} 1 keys: - 0 _col2 (type: int), _col1 (type: int) - 1 _col8 (type: int), _col7 (type: int) + 0 _col1 (type: int), _col2 (type: int) + 1 _col6 (type: int), _col7 (type: int) Map Join Vectorization: - bigTableFilterExpressions: SelectColumnIsTrue(col 33:boolean), SelectColumnIsTrue(col 34:boolean) - bigTableKeyColumns: 3:int, 2:int + bigTableFilterExpressions: SelectColumnIsTrue(col 32:boolean), SelectColumnIsTrue(col 33:boolean) + bigTableKeyColumns: 2:int, 3:int bigTableRetainColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] bigTableValueColumns: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) className: VectorMapJoinOuterMultiKeyOperator native: true nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - outerSmallTableKeyMapping: 2 -> 42, 3 -> 43 - projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) - smallTableValueMapping: 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + outerSmallTableKeyMapping: 2 -> 40, 3 -> 41 + projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) + smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) hashTableImplementationType: OPTIMIZED - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50 input vertices: 1 Map 5 Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col40 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col33 (type: int), _col39 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col38 (type: int), _col28 (type: string), _col6 (type: int), _col50 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col34 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col45 (type: decimal(7,2)), _col47 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col35 (type: int), _col31 (type: int), _col41 (type: decimal(7,2)), _col36 (type: int), _col4 (type: int), _col48 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col29 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col30 (type: int), _col42 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col51 (type: decimal(7,2)), _col32 (type: int), _col37 (type: int), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49 + expressions: _col39 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col32 (type: int), _col38 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col37 (type: int), _col6 (type: int), _col49 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col33 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col46 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col34 (type: int), _col30 (type: int), _col40 (type: decimal(7,2)), _col35 (type: int), _col4 (type: int), _col47 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col42 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col28 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col29 (type: int), _col41 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col50 (type: decimal(7,2)), _col31 (type: int), _col36 (type: int), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [51, 20, 44, 50, 35, 22, 37, 8, 4, 11, 60, 49, 39, 7, 61, 12, 45, 18, 15, 56, 58, 21, 9, 46, 42, 52, 47, 5, 59, 6, 14, 13, 55, 54, 2, 17, 40, 3, 16, 19, 36, 10, 41, 53, 1, 38, 62, 43, 48, 57] + projectedOutputColumnNums: [49, 20, 42, 48, 34, 22, 36, 8, 4, 11, 58, 47, 7, 59, 12, 43, 18, 15, 54, 56, 21, 9, 44, 40, 50, 45, 5, 57, 6, 14, 13, 53, 52, 2, 17, 38, 3, 16, 19, 35, 10, 39, 51, 1, 37, 60, 41, 46, 55] Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint), _col12 (type: string), 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), null (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint), 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), null (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [35, 36, 37, 38, 39, 31, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 63, 55, 56, 57, 58, 59, 60, 61, 62] - selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val null) -> 63:decimal(7,2) + projectedOutputColumnNums: [34, 35, 36, 37, 30, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 61, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 61:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -412,17 +412,17 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [32, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 64, 55, 56, 57, 58, 59, 60, 61, 62] - selectExpressions: ConstantVectorExpression(val 2451181) -> 32:int, ConstantVectorExpression(val 0) -> 64:decimal(7,2) + projectedOutputColumnNums: [31, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 62, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 62:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -431,28 +431,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 43:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int + keyColumns: 41:int, 64:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 32:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 64:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + valueColumns: 31:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 62:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 43:int), SelectColumnIsNull(col 40:int)) - predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 40:int), SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 38:int)) + predicate: (_col23 is null and _col46 is null and _col35 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col44 (type: int), _col34 (type: int), _col37 (type: int), _col8 (type: int), _col27 (type: int), _col29 (type: int), _col13 (type: int), _col7 (type: int), _col22 (type: int), _col41 (type: int), _col9 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col35 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col39 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col5 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col43 (type: int), _col33 (type: int), _col36 (type: int), _col8 (type: int), _col26 (type: int), _col28 (type: int), _col12 (type: int), _col7 (type: int), _col21 (type: int), _col40 (type: int), _col9 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col29 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col37 (type: decimal(7,2)), _col34 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 65:int + projectedOutputColumnNums: [63, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 63:int Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -461,27 +461,27 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 67:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 67:int + keyColumns: 3:int, 65:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 65:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 63:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col6 (type: string), _col40 (type: bigint), _col45 (type: bigint) - outputColumnNames: _col4, _col6, _col40, _col45 + expressions: _col4 (type: int), _col6 (type: string), _col39 (type: bigint), _col44 (type: bigint) + outputColumnNames: _col4, _col6, _col39, _col44 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [35, 37, 36, 38] + projectedOutputColumnNums: [34, 36, 35, 37] Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -489,11 +489,11 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 35:int, col 36:bigint, col 37:string, col 38:bigint + keyExpressions: col 34:int, col 35:bigint, col 36:string, col 37:bigint native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] - keys: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint) + keys: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -526,7 +526,7 @@ STAGE PLANS: includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] Map 5 Map Operator Tree: TableScan @@ -535,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Filter Operator Filter Vectorization: className: VectorFilterOperator @@ -544,28 +544,28 @@ STAGE PLANS: predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [23, 24, 25, 26, 27, 29, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 29:int + projectedOutputColumnNums: [23, 24, 25, 26, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col8 (type: int), _col7 (type: int) + key expressions: _col6 (type: int), _col7 (type: int) null sort order: zz sort order: ++ - Map-reduce partition columns: _col7 (type: int) + Map-reduce partition columns: _col6 (type: int) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: 3:int, 2:int + keyColumns: 2:int, 3:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int - valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 29:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: enabled: true @@ -909,16 +909,16 @@ Stage-6 Select Operator [SEL_50] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_46] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) Select Operator [SEL_44] (rows=5 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48"] Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) - BucketMapJoin:true,Conds:SEL_42._col2, _col1=RS_41._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51"] + BucketMapJoin:true,Conds:SEL_42._col1, _col2=RS_41._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] <-Map 5 [CUSTOM_EDGE] vectorized MULTICAST [RS_41] - PartitionCols:_col7 + PartitionCols:_col6 Select Operator [SEL_40] (rows=2 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"] Filter Operator [FIL_39] (rows=2 width=#Masked#) predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) TableScan [TS_2] (rows=2 width=#Masked#) @@ -938,7 +938,7 @@ Stage-6 Select Operator [SEL_51] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_47] (rows=1 width=#Masked#) - predicate:(_col24 is null and _col47 is null and _col36 is null) + predicate:(_col23 is null and _col46 is null and _col35 is null) Please refer to the previous Select Operator [SEL_44] Reducer 4 vectorized File Output Operator [FS_65] @@ -953,11 +953,11 @@ Stage-6 SHUFFLE [RS_57] PartitionCols:_col0, _col1, _col2, _col3 Group By Operator [GBY_56] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col40, _col6, _col45 + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col39, _col6, _col44 Select Operator [SEL_52] (rows=1 width=#Masked#) - Output:["_col4","_col6","_col40","_col45"] + Output:["_col4","_col6","_col39","_col44"] Filter Operator [FIL_48] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) Please refer to the previous Select Operator [SEL_44] Stage-7 Stats Work{} diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index e4275452b154..f94d7275f651 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -197,7 +197,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -351,7 +351,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -360,7 +360,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -476,7 +476,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -797,7 +797,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 3e4c4a1e6e0a..734ac8f8f824 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -102,7 +102,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -282,7 +282,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -291,7 +291,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -442,7 +442,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -576,7 +576,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -688,7 +688,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -861,7 +861,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index 71cff814d471..c2a1c1c06d12 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -280,7 +280,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -480,7 +480,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out index 37f0589a9167..02111ac58daf 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out @@ -239,8 +239,8 @@ STAGE PLANS: predicate: (a = 22) (type: boolean) Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -321,12 +321,12 @@ STAGE PLANS: predicate: (c = 66) (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col9 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col8 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), 66 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), 66 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -337,7 +337,7 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Select Operator - expressions: 33 (type: int), _col9 (type: string), 66 (type: int) + expressions: 33 (type: int), _col8 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -456,16 +456,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -474,19 +474,19 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -497,11 +497,11 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -512,10 +512,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -527,10 +527,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: _col10 is null (type: boolean) + predicate: _col9 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -542,15 +542,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: (_col10 = _col1) (type: boolean) + predicate: (_col9 = _col1) (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 64bd708c37ea..b07470fb5374 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -680,12 +680,6 @@ public static Object[] populateVirtualColumnValues(ExecMapperContext ctx, vcValues[i] = new LongWritable(ctx.getIoCxt().getPositionDeleteInfo().getPartitionHash()); } break; - case PARTITION_PROJECTION: - vcValues[i] = null; - if (ctx.getIoCxt().getPositionDeleteInfo() != null) { - vcValues[i] = new Text(ctx.getIoCxt().getPositionDeleteInfo().getPartitionProjection()); - } - break; case PARTITION_NAME: vcValues[i] = null; if (ctx.getIoCxt().getPartitionName() != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java index f5e531496734..d3485a49f713 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java @@ -27,38 +27,32 @@ public class PositionDeleteInfo { private static final String CONF_KEY_PART_HASH = "hive.io.context.position.delete.partition.hash"; private static final String CONF_KEY_FILE_PATH = "hive.io.context.position.delete.file.path"; private static final String CONF_KEY_ROW_POSITION = "hive.io.context.position.delete.row.position"; - private static final String CONF_KEY_PARTITION_PROJECTION = "hive.io.context.position.delete.partition.projection"; public static PositionDeleteInfo parseFromConf(Configuration conf) { int specId = conf.getInt(CONF_KEY_SPEC_ID, -1); long partHash = conf.getLong(CONF_KEY_PART_HASH, -1); String filePath = conf.get(CONF_KEY_FILE_PATH); long rowPos = conf.getLong(CONF_KEY_ROW_POSITION, -1); - String partitionProjection = conf.get(CONF_KEY_PARTITION_PROJECTION); - return new PositionDeleteInfo(specId, partHash, filePath, rowPos, partitionProjection); + return new PositionDeleteInfo(specId, partHash, filePath, rowPos); } - public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos, - String partitionProjection) { + public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos) { conf.setInt(CONF_KEY_SPEC_ID, specId); conf.setLong(CONF_KEY_PART_HASH, partHash); conf.set(CONF_KEY_FILE_PATH, filePath); conf.setLong(CONF_KEY_ROW_POSITION, filePos); - conf.set(CONF_KEY_PARTITION_PROJECTION, partitionProjection); } private final int specId; private final long partitionHash; private final String filePath; private final long filePos; - private final String partitionProjection; - public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos, String partitionProjection) { + public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos) { this.specId = specId; this.partitionHash = partitionHash; this.filePath = filePath; this.filePos = filePos; - this.partitionProjection = partitionProjection; } public int getSpecId() { @@ -76,8 +70,4 @@ public String getFilePath() { public long getFilePos() { return filePos; } - - public String getPartitionProjection() { - return partitionProjection; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index fb5852cf144c..4fd718af61be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -57,7 +57,6 @@ public enum VirtualColumn { FILE_PATH("FILE__PATH", TypeInfoFactory.stringTypeInfo), ROW_POSITION("ROW__POSITION", TypeInfoFactory.longTypeInfo), SNAPSHOT_ID("SNAPSHOT__ID", TypeInfoFactory.longTypeInfo), - PARTITION_PROJECTION("PARTITION__PROJECTION", TypeInfoFactory.stringTypeInfo), PARTITION_NAME("PARTITION__NAME", TypeInfoFactory.stringTypeInfo), ROW_LINEAGE_ID("ROW__LINEAGE__ID", TypeInfoFactory.longTypeInfo), LAST_UPDATED_SEQUENCE_NUMBER("LAST__UPDATED__SEQUENCE__NUMBER", TypeInfoFactory.longTypeInfo), @@ -75,7 +74,7 @@ public enum VirtualColumn { ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName(), ROWISDELETED.getName(), PARTITION_SPEC_ID.getName(), PARTITION_HASH.getName(), FILE_PATH.getName(), ROW_POSITION.getName(), - PARTITION_PROJECTION.getName(), PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), + PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), LAST_UPDATED_SEQUENCE_NUMBER.getName()); public static final ImmutableMap VIRTUAL_COLUMN_NAME_MAP = diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 7d0a3cb2a8cc..12c43f7f6e8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -309,7 +309,6 @@ public class Vectorizer implements PhysicalPlanResolver { VirtualColumn.PARTITION_HASH, VirtualColumn.FILE_PATH, VirtualColumn.ROW_POSITION, - VirtualColumn.PARTITION_PROJECTION, VirtualColumn.PARTITION_NAME, VirtualColumn.ROW_LINEAGE_ID, VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); From 472d6baf86ae34c5f99735a96447411426fdfab4 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 24 Aug 2026 15:35:49 +0300 Subject: [PATCH 03/15] HIVE-29829: Gather Iceberg column statistics in one scan, and settle each write in one place A partition scoped gather ran one query per partition spec, each grouped by that spec's own transforms. The rows of every spec already carry the name of the partition they belong to, so one scan grouped by that name gathers all of them, and the map of specs it took to drive the old shape is gone. What a write then does to the stored statistics - replace them, merge into them, or leave them alone - was settled across writeColStats, shouldRewriteColStats, canProvideColStats and the merge itself. ColStatsWritePolicy now states every case of that decision over the facts of the write alone, so a test can cover it without a table. --- .../apache/iceberg/hive/MetastoreUtil.java | 27 +- .../mr/hive/HiveIcebergStorageHandler.java | 222 +--- .../iceberg/mr/hive/IcebergTableUtil.java | 239 +++- .../compaction/IcebergCompactionService.java | 4 +- .../mr/hive/stats/ColStatsWritePolicy.java | 119 ++ .../mr/hive/TestHiveIcebergStatistics.java | 929 ++++++++++++- .../results/positive/bucket_map_join_9.q.out | 18 +- .../src/test/results/positive/col_stats.q.out | 2 +- .../ctas_iceberg_partitioned_orc.q.out | 2 +- ...te_iceberg_copy_on_write_partitioned.q.out | 142 +- ..._iceberg_copy_on_write_unpartitioned.q.out | 340 ++--- .../positive/delete_iceberg_mixed.q.out | 36 +- .../positive/dynamic_partition_pruning.q.out | 608 ++++----- .../positive/dynamic_partition_writes.q.out | 511 ++++---- .../positive/dynamic_semijoin_reduction.q.out | 42 +- .../iceberg_insert_into_partition.q.out | 1152 ++++++++--------- ...erg_insert_into_partition_transforms.q.out | 692 +++++----- ...insert_into_partition_with_evolution.q.out | 32 +- .../iceberg_insert_overwrite_partition.q.out | 794 +++++------- ...nsert_overwrite_partition_transforms.q.out | 558 ++++---- .../results/positive/iceberg_load_data.q.out | 42 +- .../positive/iceberg_pcr_null_partition.q.out | 12 +- .../positive/iceberg_stats_with_ppr.q.out | 6 +- ...rg_truncate_partition_with_evolution.q.out | 32 +- .../llap/hadoop_catalog_create_table.q.out | 6 +- .../llap/iceberg_bucket_map_join_2.q.out | 52 +- .../llap/iceberg_bucket_map_join_3.q.out | 36 +- .../llap/iceberg_bucket_map_join_4.q.out | 50 +- .../llap/iceberg_bucket_map_join_5.q.out | 36 +- .../llap/iceberg_bucket_map_join_6.q.out | 26 +- .../llap/iceberg_bucket_map_join_7.q.out | 334 ++--- .../llap/iceberg_bucket_map_join_8.q.out | 70 +- ...ceberg_create_locally_zordered_table.q.out | 30 +- ...major_compaction_partition_evolution.q.out | 26 +- ...ajor_compaction_partition_evolution2.q.out | 6 +- ...mpaction_partition_evolution_ordered.q.out | 4 +- ...ion_single_partition_with_evolution2.q.out | 2 +- .../iceberg_minor_compaction_bucket.q.out | 4 +- ...minor_compaction_partition_evolution.q.out | 4 +- ...eberg_minor_compaction_unpartitioned.q.out | 2 +- .../positive/llap/llap_iceberg_read_orc.q.out | 24 +- ...ge_iceberg_copy_on_write_partitioned.q.out | 80 +- .../merge_iceberg_partitioned_orc.q.out | 56 +- ...ceberg_metadata_of_partitioned_table.q.out | 8 - .../src/test/results/positive/row_count.q.out | 23 +- ...te_iceberg_copy_on_write_partitioned.q.out | 218 ++-- ..._iceberg_copy_on_write_unpartitioned.q.out | 392 +++--- .../update_iceberg_partitioned_avro.q.out | 4 +- .../positive/write_iceberg_branch.q.out | 82 +- .../table/info/desc/DescTableOperation.java | 9 +- .../apache/hadoop/hive/ql/exec/StatsTask.java | 5 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 14 +- .../hive/ql/metadata/HiveStorageHandler.java | 3 - .../apache/hadoop/hive/ql/metadata/Table.java | 4 +- .../hive/ql/optimizer/GenMapRedUtils.java | 5 + .../parse/ColumnStatsAutoGatherContext.java | 9 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 84 +- .../hadoop/hive/ql/parse/TransformSpec.java | 16 - .../hadoop/hive/ql/stats/BasicStatsTask.java | 13 +- .../hive/ql/stats/ColStatsProcessor.java | 39 +- .../hadoop/hive/ql/stats/StatsUtils.java | 32 +- 61 files changed, 4525 insertions(+), 3844 deletions(-) create mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index d652dbc6ef51..dcd44b6033ab 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -48,6 +47,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.SQLViewRepresentation; @@ -119,17 +119,18 @@ public static void alterTable( } } - public static List getPartitionKeys(org.apache.iceberg.Table table, int specId) { - Schema schema = table.specs().get(specId).schema(); - List hiveSchema = HiveSchemaUtil.convert(schema); - Map colNameToColType = hiveSchema.stream() - .collect(Collectors.toMap(FieldSchema::getName, FieldSchema::getType)); - return table.specs().get(specId).fields().stream() - .map(partField -> new FieldSchema( - schema.findColumnName(partField.sourceId()), - colNameToColType.get(schema.findColumnName(partField.sourceId())), - String.format("Transform: %s", partField.transform().toString())) - ) + public static List getPartitionKeys(org.apache.iceberg.Table table) { + Schema schema = table.spec().schema(); + + return table.spec().fields().stream() + .map(partField -> { + Types.NestedField col = schema.findField(partField.sourceId()); + return new FieldSchema( + col.name().toLowerCase(), // HMS lowercases column names + HiveSchemaUtil.convertToTypeString(col.type()), + "Transform: %s".formatted(partField.transform()) + ); + }) .toList(); } @@ -143,7 +144,7 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co result.setTableType(TableType.EXTERNAL_TABLE.toString()); // TODO: Revert after HIVE-29633 is fixed - // result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); + // result.setPartitionKeys(getPartitionKeys(table)); result.setPartitionKeys(Lists.newArrayList()); TableMetadata metadata = ((BaseTable) table).operations().current(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index d79275668a67..429bdcd4a3aa 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -20,11 +20,9 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.io.Serializable; import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -44,19 +42,16 @@ import java.util.stream.Stream; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.SnapshotContext; import org.apache.hadoop.hive.common.type.Timestamp; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -142,8 +137,6 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.FindFiles; -import org.apache.iceberg.GenericBlobMetadata; -import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionData; @@ -188,12 +181,9 @@ import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.hive.actions.HiveIcebergDeleteOrphanFiles; import org.apache.iceberg.mr.hive.plan.IcebergBucketFunction; +import org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy; import org.apache.iceberg.mr.hive.udf.GenericUDFIcebergZorder; -import org.apache.iceberg.puffin.Blob; import org.apache.iceberg.puffin.BlobMetadata; -import org.apache.iceberg.puffin.Puffin; -import org.apache.iceberg.puffin.PuffinCompressionCodec; -import org.apache.iceberg.puffin.PuffinWriter; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; @@ -205,7 +195,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.SerializationUtil; import org.apache.iceberg.util.SnapshotUtil; import org.slf4j.Logger; @@ -236,9 +225,8 @@ public class HiveIcebergStorageHandler extends DefaultStorageHandler implements private static final String TABLE_NAME_SEPARATOR = ".."; public static final String TABLE_DEFAULT_LOCATION = "TABLE_DEFAULT_LOCATION"; - private static final String PARTITION = "partition"; + private static final String PARTITION = IcebergTableUtil.PARTITION_FIELD; private static final String PARTITION_STATS_PREFIX = "partitionStats."; - public static final String STATS = "/stats/snap-"; public static final String COPY_ON_WRITE = RowLevelOperationMode.COPY_ON_WRITE.modeName(); public static final String MERGE_ON_READ = RowLevelOperationMode.MERGE_ON_READ.modeName(); @@ -521,7 +509,9 @@ private Map getBasicStatistics(org.apache.hadoop.hive.ql.metadat if (snapshot == null) { stats = emptyStatsMap(); - } else if (!HiveMetaHook.ICEBERG.equals(getStatsSource()) && !quickStats) { + } else if (!HiveMetaHook.ICEBERG.equals(getStatsSource()) && !quickStats && + hmsTable.getSnapshotRef() == null) { + // the metastore parameters describe the table, not a branch: use the snapshot's counters stats = hmsTable.getParameters(); } else { @@ -628,7 +618,7 @@ public Map computeBasicStatistics(org.apache.hadoop.hive.ql.meta .commit(); statsTable = tx.table(); } - statsFile = PartitionStatsHandler.computeAndWriteStatsFile(statsTable); + statsFile = PartitionStatsHandler.computeAndWriteStatsFile(statsTable, snapshot.snapshotId()); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -727,116 +717,38 @@ public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsT @Override public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colStats) { Table tbl = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return writeColStats(colStats, tbl); - } - - @SuppressWarnings("checkstyle:CyclomaticComplexity") - private boolean writeColStats(List colStats, Table tbl) { + // a write to a branch moves that branch's head, leaving the table's current snapshot behind + Snapshot snapshot = IcebergTableUtil.getTableSnapshot(tbl, hmsTable); + if (snapshot == null || colStats.isEmpty()) { + return false; + } + ColStatsWritePolicy policy = ColStatsWritePolicy.resolve(tbl, snapshot, colStats, conf); + if (policy == ColStatsWritePolicy.SKIP) { + return IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf); + } try { - if (!shouldRewriteColStats(tbl)) { - checkAndMergeColStats(colStats, tbl); - } - StatisticsFile statisticsFile; - String statsPath = tbl.location() + STATS + UUID.randomUUID(); - - try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) - .createdBy(Constants.HIVE_ENGINE) - .build()) { - - long snapshotId = tbl.currentSnapshot().snapshotId(); - long snapshotSequenceNumber = tbl.currentSnapshot().sequenceNumber(); - Schema schema = tbl.spec().schema(); - - boolean first = true; - - for (ColumnStatistics stats : colStats) { - boolean isTblLevel = stats.getStatsDesc().isIsTblLevel(); - - Map properties = isTblLevel ? Map.of() : - Map.of(PARTITION, String.valueOf(stats.getStatsDesc().getPartName())); - - List statsObjects = isTblLevel ? - stats.getStatsObj() : List.of(stats); - - List fieldIds = null; - - if (!isTblLevel) { - // For partition-level stats, we emit one blob per partition; - // therefore, only the first blob should contain the actual fieldIds. - fieldIds = !first ? List.of(-1) : - stats.getStatsObj().stream() - .map(obj -> schema.findField(obj.getColName()).fieldId()) - .toList(); - first = false; - } - - for (Serializable statsObj : statsObjects) { - byte[] serialized = SerializationUtils.serialize(statsObj); - - if (isTblLevel) { - fieldIds = List.of(schema.findField( - ((ColumnStatisticsObj) statsObj).getColName()).fieldId()); - } - - writer.add(new Blob( - ColumnStatisticsObj.class.getSimpleName(), - fieldIds, - snapshotId, - snapshotSequenceNumber, - ByteBuffer.wrap(serialized), - PuffinCompressionCodec.NONE, - properties - )); - } - } - - writer.finish(); - - statisticsFile = - new GenericStatisticsFile( - snapshotId, - statsPath, - writer.fileSize(), - writer.footerSize(), - writer.writtenBlobsMetadata().stream() - .map(GenericBlobMetadata::from) - .collect(ImmutableList.toImmutableList()) - ); - } catch (IOException e) { - LOG.warn("Unable to write column stats to the Puffin file: {}", e.getMessage()); - - Path path = new Path(statsPath); - FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - fs.delete(path, false); - } + if (policy == ColStatsWritePolicy.MERGE && !checkAndMergeColStats(colStats, tbl, snapshot)) { return false; } - tbl.updateStatistics() - .setStatistics(statisticsFile) - .commit(); - return true; - } catch (Exception e) { - LOG.warn("Unable to invalidate or merge column stats: {}", e.getMessage()); + LOG.warn("Unable to merge column stats: {}", e.getMessage()); + return false; } - return false; + return IcebergTableUtil.writeColStats(tbl, snapshot, colStats, conf); } + @Override public boolean canProvideColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); if (snapshot != null) { - return canSetColStatistics(hmsTable) && canProvideColStats(table, snapshot.snapshotId()); + return canSetColStatistics(hmsTable) && + IcebergTableUtil.findColStatsFile(table, snapshot.snapshotId(), conf) != null; } return false; } - private boolean canProvideColStats(Table table, long snapshotId) { - return IcebergTableUtil.getColStatsPath(table, snapshotId) != null; - } - @Override public List getColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colNames) { @@ -864,7 +776,7 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta filter = null; } - return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); + return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), conf, filter); } @Override @@ -882,9 +794,10 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms double ndvTuner = MetastoreConf.getDoubleVar(getConf(), MetastoreConf.ConfVars.STATS_NDV_TUNER); Set partitions = Sets.newHashSet(partNames); - Predicate filter = metadata -> partitions.contains(metadata.properties().get(PARTITION)); + Predicate filter = metadata -> + partitions.contains(metadata.properties().get(PARTITION)); - List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); + List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), conf, filter); partStats.forEach(colStats -> colStats.getStatsObj().removeIf(statsObj -> !colNames.contains(statsObj.getColName()))); @@ -903,7 +816,7 @@ public Long getRowCount(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { if (hmsTable.getMetaTable() != null) { return null; } - return getStatsSource().equals(HiveMetaHook.ICEBERG) ? + return getStatsSource().equals(HiveMetaHook.ICEBERG) || hmsTable.getSnapshotRef() != null ? snapshotRowCount(hmsTable) : metastoreRowCount(hmsTable); } @@ -955,46 +868,30 @@ private String getStatsSource() { .toUpperCase(); } - private boolean shouldRewriteColStats(Table tbl) { - return SessionStateUtil.getQueryState(conf) - .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) - .orElse(false) || - IcebergTableUtil.getColStatsPath(tbl) != null; - } - - private void checkAndMergeColStats(List statsNew, Table tbl) throws InvalidObjectException { - Long previousSnapshotId = tbl.currentSnapshot().parentId(); - if (previousSnapshotId != null && canProvideColStats(tbl, previousSnapshotId)) { - - boolean isTblLevel = statsNew.getFirst().getStatsDesc().isIsTblLevel(); - Map oldStatsMap = Maps.newHashMap(); - - List statsOld = IcebergTableUtil.readColStats(tbl, previousSnapshotId, null); - if (!isTblLevel) { - for (ColumnStatistics statsObjOld : (List) statsOld) { - oldStatsMap.put(statsObjOld.getStatsDesc().getPartName(), statsObjOld); - } - } else { - statsOld = Collections.singletonList( - new ColumnStatistics(null, (List) statsOld)); - } - for (ColumnStatistics statsObjNew : statsNew) { - String partitionKey = statsObjNew.getStatsDesc().getPartName(); - ColumnStatistics statsObjOld = isTblLevel ? - (ColumnStatistics) statsOld.getFirst() : oldStatsMap.get(partitionKey); - - if (statsObjOld != null && statsObjOld.getStatsObjSize() != 0 && !statsObjNew.getStatsObj().isEmpty()) { - MetaStoreServerUtils.mergeColStats(statsObjNew, statsObjOld); - if (!isTblLevel) { - oldStatsMap.remove(partitionKey); - } - } - } - if (!isTblLevel) { - statsNew.addAll(oldStatsMap.values()); - } + /** + * Extends a table-level increment with the previous snapshot's statistics - or lets it stand + * alone when no rows precede it (first write, truncate). Returns whether the write may proceed: + * without previous statistics to extend, the increment is dropped - only a complete rewrite + * can restart the chain. + */ + private boolean checkAndMergeColStats(List statsNew, Table tbl, Snapshot snapshot) + throws InvalidObjectException, IOException { + Long prevSnapshotId = snapshot.parentId(); + if (prevSnapshotId == null || IcebergTableUtil.isEmptySnapshot(tbl.snapshot(prevSnapshotId))) { + // an empty base: the increment is the complete statistics + return true; + } + StatisticsFile statsOldSrc = IcebergTableUtil.getColStatsFile(tbl, prevSnapshotId, conf); + if (statsOldSrc == null) { + return false; + } + List statsOld = IcebergTableUtil.readColStatsOrThrow(tbl, statsOldSrc, null); + ColumnStatistics statsObjOld = new ColumnStatistics(null, statsOld); + if (statsObjOld.getStatsObjSize() != 0 && !statsNew.getFirst().getStatsObj().isEmpty()) { + MetaStoreServerUtils.mergeColStats(statsNew.getFirst(), statsObjOld); } + return true; } /** @@ -1037,30 +934,11 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m .filter(f -> !f.transform().isVoid()) .map(f -> { TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); - spec.setFieldName(f.name()); return spec; }) .collect(Collectors.toList()); } - @Override - public Map> getPartitionTransformSpecs( - org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) { - return Collections.emptyMap(); - } - Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return table.specs().entrySet().stream().flatMap(e -> - e.getValue().fields().stream() - .filter(f -> !f.transform().isVoid()) - .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); - spec.setFieldName(f.name()); - return Pair.of(e.getKey(), spec); - })) - .collect(Collectors.groupingBy( - Pair::first, Collectors.mapping(Pair::second, Collectors.toList()))); - } private List getWriteSortTransformSpecs(Table table) { return table.sortOrder().fields().stream() @@ -2463,7 +2341,7 @@ public List getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Tab return hmsTable.getPartitionKeys(); } Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return MetastoreUtil.getPartitionKeys(icebergTable, icebergTable.spec().specId()); + return MetastoreUtil.getPartitionKeys(icebergTable); } @Override diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 728c69f98f7d..1fe44a0aee5f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -20,6 +20,7 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; +import java.io.Serializable; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.time.ZoneId; @@ -34,6 +35,7 @@ import java.util.Properties; import java.util.Set; import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; @@ -43,13 +45,18 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -71,9 +78,12 @@ import org.apache.hadoop.util.Sets; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataOperations; import org.apache.iceberg.DeleteFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.ManifestFiles; @@ -90,6 +100,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; @@ -106,10 +117,14 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.puffin.Blob; import org.apache.iceberg.puffin.BlobMetadata; import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinCompressionCodec; import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.puffin.PuffinWriter; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -142,6 +157,9 @@ public class IcebergTableUtil { private static final String SPEC_ID_FIELD = "spec_id"; private static final String NULL_VALUE = "null"; + static final String PARTITION_FIELD = "partition"; + static final String STATS = "/stats/snap-"; + private IcebergTableUtil() { } @@ -253,18 +271,80 @@ static Snapshot getTableSnapshot(Table table, String snapshotRef) { return table.currentSnapshot(); } - static String getColStatsPath(Table table) { - return getColStatsPath(table, table.currentSnapshot().snapshotId()); + /** The current snapshot's own column statistics file of the flag-maintained granularity. */ + static StatisticsFile getColStatsFile(Table table, Configuration conf) { + return lookupColStatsFile(table, table.currentSnapshot().snapshotId(), conf); + } + + /** + * The newest column statistics file describing the snapshot: its own, or - across commits that + * rewrite files without changing any rows (compaction) - an ancestor's. Only a file of the + * granularity the flag maintains is served: partition-level blobs carry the partition name in + * their metadata, table-level blobs don't. + */ + public static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { + for (Snapshot snapshot = table.snapshot(snapshotId); snapshot != null; + snapshot = snapshot.parentId() != null ? table.snapshot(snapshot.parentId()) : null) { + StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), conf); + if (statsFile != null) { + return statsFile; + } + if (!DataOperations.REPLACE.equals(snapshot.operation())) { + return null; + } + } + return null; + } + + /** + * The nearest column statistics file describing the snapshot: its own, or the closest + * ancestor's. The lookup stops at a snapshot holding no rows (truncate): earlier statistics + * describe data that didn't survive it. Statistics of an ancestor describe an earlier state of + * the data, which {@link #colStatsAccurate} reports and the planner treats as partial. + */ + static StatisticsFile findColStatsFile(Table table, long snapshotId, Configuration conf) { + if (table.statisticsFiles().isEmpty()) { + return null; + } + for (Snapshot snapshot = table.snapshot(snapshotId); snapshot != null; + snapshot = snapshot.parentId() != null ? table.snapshot(snapshot.parentId()) : null) { + StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), conf); + if (statsFile != null) { + return statsFile; + } + if (isEmptySnapshot(snapshot)) { + return null; + } + } + return null; } - static String getColStatsPath(Table table, long snapshotId) { + /** + * Only a file of the granularity the flag maintains is served: partition-level blobs carry + * the partition name in their metadata, table-level blobs don't. + */ + private static StatisticsFile lookupColStatsFile(Table table, long snapshotId, Configuration conf) { return table.statisticsFiles().stream() - .filter(stats -> stats.snapshotId() == snapshotId) - .filter(stats -> stats.blobMetadata().stream() - .anyMatch(metadata -> ColumnStatisticsObj.class.getSimpleName().equals(metadata.type())) - ) - .map(StatisticsFile::path) - .findAny().orElse(null); + .filter(stats -> stats.snapshotId() == snapshotId) + .filter(stats -> stats.blobMetadata().stream().anyMatch(metadata -> + isHiveColStatsBlob(table, metadata, conf))) + .findAny().orElse(null); + } + + /** Whether the snapshot holds no rows. */ + public static boolean isEmptySnapshot(Snapshot snapshot) { + return snapshot != null && snapshot.summary() != null && + NumberUtils.toLong(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP), -1) == 0; + } + + /** Whether the table's column statistics are maintained per partition, as the flag directs. */ + public static boolean isPartitionStats(Table table, Configuration conf) { + return HiveConf.getBoolVar(conf, ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && table.spec().isPartitioned(); + } + + private static boolean isHiveColStatsBlob(Table table, org.apache.iceberg.BlobMetadata metadata, Configuration conf) { + return ColumnStatisticsObj.class.getSimpleName().equals(metadata.type()) && + isPartitionStats(table, conf) == metadata.properties().containsKey(PARTITION_FIELD); } static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotId) { @@ -804,15 +884,142 @@ public static TransformSpec getTransformSpec(Table table, String transformName, return spec; } - public static List readColStats(Table table, Long snapshotId, Predicate filter) { - List colStats = Lists.newArrayList(); + /** + * Writes the column statistics as the table's statistics file for the current snapshot: one blob + * per column at table level, one blob per partition otherwise, each carrying the partition name + * the read side joins on. + */ + @SuppressWarnings("checkstyle:CyclomaticComplexity") + static boolean writeColStats(Table tbl, Snapshot snapshot, List colStats, Configuration conf) { + try { + StatisticsFile statisticsFile; + String statsPath = tbl.location() + STATS + UUID.randomUUID(); + + try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) + .createdBy(Constants.HIVE_ENGINE) + .build()) { + + long snapshotId = snapshot.snapshotId(); + long snapshotSequenceNumber = snapshot.sequenceNumber(); + Schema schema = tbl.spec().schema(); + + boolean first = true; + + for (ColumnStatistics stats : colStats) { + boolean isTblLevel = stats.getStatsDesc().isIsTblLevel(); + + Map properties = isTblLevel ? Map.of() : + Map.of(PARTITION_FIELD, String.valueOf(stats.getStatsDesc().getPartName())); - String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); - if (statsPath == null) { + List statsObjects = isTblLevel ? + stats.getStatsObj() : List.of(stats); + + // a column dropped or renamed since the entry was stored resolves no field: its + // statistics leave with it + stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); + + List fieldIds = null; + + if (!isTblLevel) { + // For partition-level stats, we emit one blob per partition; + // therefore, only the first blob should contain the actual fieldIds. + fieldIds = !first ? List.of(-1) : + stats.getStatsObj().stream() + .map(obj -> schema.caseInsensitiveFindField(obj.getColName()).fieldId()) + .toList(); + first = false; + } + + for (Serializable statsObj : statsObjects) { + byte[] serialized = SerializationUtils.serialize(statsObj); + + if (isTblLevel) { + fieldIds = List.of(schema.caseInsensitiveFindField( + ((ColumnStatisticsObj) statsObj).getColName()).fieldId()); + } + + writer.add(new Blob( + ColumnStatisticsObj.class.getSimpleName(), + fieldIds, + snapshotId, + snapshotSequenceNumber, + ByteBuffer.wrap(serialized), + PuffinCompressionCodec.NONE, + properties + )); + } + } + + writer.finish(); + + statisticsFile = + new GenericStatisticsFile( + snapshotId, + statsPath, + writer.fileSize(), + writer.footerSize(), + writer.writtenBlobsMetadata().stream() + .map(GenericBlobMetadata::from) + .collect(ImmutableList.toImmutableList()) + ); + } catch (IOException e) { + LOG.warn("Unable to write column stats to the Puffin file: {}", e.getMessage()); + + Path path = new Path(statsPath); + FileSystem fs = path.getFileSystem(conf); + if (fs.exists(path)) { + fs.delete(path, false); + } + return false; + } + tbl.updateStatistics() + .setStatistics(statisticsFile) + .commit(); + return true; + + } catch (Exception e) { + LOG.warn("Unable to write column stats: {}", e.getMessage()); + } + return false; + } + + /** + * Whether the stored column statistics still describe the table: the current snapshot owns them, + * or only row-preserving commits (compaction) separate it from the snapshot that does. Derived + * from the table metadata, so it holds for the writes of every engine. + */ + public static boolean colStatsAccurate(Table table, Snapshot snapshot, Configuration conf) { + return getColStatsFile(table, snapshot.snapshotId(), conf) != null; + } + + static List readColStats(Table table, long snapshotId, Configuration conf, + Predicate filter) { + StatisticsFile statsFile = findColStatsFile(table, snapshotId, conf); + if (statsFile == null) { LOG.warn("Column stats file not found for snapshot: {}", snapshotId); - return colStats; + return Lists.newArrayList(); + } + try { + return readColStatsOrThrow(table, statsFile, filter); + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to read column stats: {}", e.getMessage()); + return Lists.newArrayList(); } - try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)).build()) { + } + + /** + * The strict variant for the merge path: an unreadable statistics file must not be mistaken for + * an absent one, or the increment would be persisted as the complete statistics. + */ + static List readColStatsOrThrow(Table table, StatisticsFile statsFile, Predicate filter) + throws IOException { + List colStats = Lists.newArrayList(); + String statsPath = statsFile.path(); + try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)) + .withFileSize(statsFile.fileSizeInBytes()) + .withFooterSize(statsFile.fileFooterSizeInBytes()) + .build()) { List blobMetadata = reader.fileMetadata().blobs(); if (filter != null) { @@ -826,8 +1033,6 @@ public static List readColStats(Table table, Long snapshotId, Predicate colStats, Configuration conf) { + + if (!colStats.getFirst().getStatsDesc().isIsTblLevel()) { + // partition-level statistics have a single writer: a table-wide ANALYZE + return isAnalyzeOperation(conf) ? REPLACE : SKIP; + } + if (IcebergTableUtil.isPartitionStats(tbl, conf)) { + // the gather covered the rows, not the granularity the table maintains + return SKIP; + } + if (SessionState.get().isCompaction()) { + // compaction preserves rows, so accurate statistics stay exact; a whole-table major one + // has read every row already, making a refresh of stale statistics free + return isFullTableMajorCompaction(conf) && !IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf) ? + REPLACE : SKIP; + } + if (isAnalyzeOperation(conf)) { + // a full recompute, and the one statement reaching here that modifies no rows + return REPLACE; + } + if (IcebergTableUtil.isEmptySnapshot(snapshot)) { + // before the row count: an overwrite that selects nothing empties the table, writing none + return REPLACE; + } + if (wroteNoRows(conf)) { + return SKIP; + } + if (IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf)) { + // a committed write leaves its snapshot without statistics until this call writes them, so + // a snapshot already owning them saw no write here: the fallback when no query state exists + return SKIP; + } + if (Boolean.parseBoolean(snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP))) { + // a partition overwrite replaces a slice: its increment neither extends the previous + // statistics nor recomputes them, while a whole-table one recomputes them + return tbl.spec().isPartitioned() ? SKIP : REPLACE; + } + return MERGE; + } + + private static boolean isFullTableMajorCompaction(Configuration conf) { + return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)) && + conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + } + + /** + * Whether the statement wrote no rows: the column statistics are gathered by the write itself, so + * by the time they are persisted the query state carries the row count its file sink reported. + */ + private static boolean wroteNoRows(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> qs.getNumModifiedRows() == 0) + .orElse(false); + } + + private static boolean isAnalyzeOperation(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) + .orElse(false); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index df57578b7052..c60f4b6b5098 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -20,10 +20,16 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -36,13 +42,19 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DataOperations; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatistics; +import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.hadoop.ConfigProperties; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.hive.test.TestTables; import org.apache.iceberg.mr.hive.test.TestTables.TestTableType; import org.apache.iceberg.mr.hive.test.utils.HiveIcebergStorageHandlerTestUtils; @@ -50,6 +62,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.thrift.TException; import org.junit.Assert; @@ -180,6 +194,86 @@ private TableIdentifier getTableIdentifierWithPessimisticLock(String hiveLockEna return identifier; } + @Test + public void testBranchWriteLeavesTableStatsUntouched() { + // a branch write leaves the table's snapshot where it is, so the table's column and basic + // statistics must still describe the table's own rows + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + // answer from the data, not from the statistics under test + shell.setHiveSessionValue("hive.compute.query.using.stats", false); + List mainRows = shell.executeStatement("SELECT max(customer_id) FROM " + identifier); + Assert.assertEquals("the row belongs to the branch, not the table", "2", + String.valueOf(mainRows.get(0)[0])); + + // the table's column statistics still describe the table's rows + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + checkColStat(identifier.name(), "customer_id", true); + + // and the basic statistics likewise count the table's rows, not the branch's + Map basicStats = storageHandler().getBasicStatistics(hmsTable(identifier)); + Assert.assertEquals("3", basicStats.get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(3L), storageHandler().getRowCount(hmsTable(identifier))); + } + + @Test + public void testBranchWriteStoresItsColStatsOnTheBranch() { + // the statistics file is anchored to a snapshot: a branch write stores what it gathered on + // the branch's head, not on the table's + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_stats"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + Table icebergTable = testTables.loadTable(identifier); + Assert.assertNotNull("the branch's head carries the statistics its write gathered", + IcebergTableUtil.getColStatsFile( + icebergTable, icebergTable.snapshot("b1").snapshotId(), shell.getHiveConf())); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + } + + @Test + public void testAnalyzeOnBranchStoresStatsOnTheBranch() { + // an explicit ANALYZE of a branch describes the branch's rows: its statistics belong to the + // branch's head, and the table's own statistics stay as they were + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_analyze_branch"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS FOR COLUMNS"); + + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + Table icebergTable = testTables.loadTable(identifier); + Assert.assertNotNull("the branch's head carries the statistics the analyze computed", + IcebergTableUtil.getColStatsFile( + icebergTable, icebergTable.snapshot("b1").snapshotId(), shell.getHiveConf())); + } + @Test public void testStatsWithInsertOverwrite() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -188,6 +282,11 @@ public void testStatsWithInsertOverwrite() { testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + // pre-existing statistics: the overwrite must replace them, not merge onto them + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1, identifier, true); shell.executeStatement(insert); @@ -196,6 +295,37 @@ public void testStatsWithInsertOverwrite() { checkColStatMinMaxValue(identifier.name(), "customer_id", 3, 5); } + @Test + public void testStatsWithPartitionedInsertOverwrite() { + // a partition overwrite maintains no partition-level statistics: the pre-existing ones keep + // serving as approximations until ANALYZE recomputes + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_iow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (5, 'a')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + List colStats = readCurrentColStats(identifier); + // p=a recomputed: the overwritten rows no longer bound its range + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getHighValue()); + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); + } + @Test public void testStatsWithPartitionedInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -213,6 +343,8 @@ public void testStatsWithPartitionedInsert() { String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false); shell.executeStatement(insert); + // partition-level statistics are maintained by complete-scope writers, not inserts + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); checkColStat(identifier.name(), "customer_id", true); checkColStat(identifier.name(), "first_name", true); @@ -360,7 +492,8 @@ public void testIcebergColStatsPath() throws IOException { table.refresh(); - Path tblColPath = new Path(IcebergTableUtil.getColStatsPath(table)); + Path tblColPath = new Path(IcebergTableUtil.getColStatsFile( + table, table.currentSnapshot().snapshotId(), shell.getHiveConf()).path()); Assert.assertNotNull(tblColPath); // Check that if colPath is created correctly Assert.assertTrue(tblColPath.getFileSystem(shell.getHiveConf()).exists(tblColPath)); @@ -370,7 +503,7 @@ public void testIcebergColStatsPath() throws IOException { } @Test - public void testGetAggrBasicStatsForPartitioned() { + public void testGetAggrBasicStatsForPartitioned() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -404,27 +537,25 @@ public void testAnalyzePartitionSpecRejected() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); createPartitionedCustomers(identifier); - String expected = ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(); - // basic statistics are maintained incrementally for all partitions as a whole: - // a partition-scoped basic-stats ANALYZE cannot be honored and must be rejected + // statistics are maintained for all partitions as a whole: a partition-scoped ANALYZE is + // rejected for non-native tables, for basic and column statistics alike AssertHelpers.assertThrows( "Should reject partition-scoped basic-stats ANALYZE for non-native partitioned tables", - IllegalArgumentException.class, expected, + IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS") ); - // same for column statistics: the rewrite would drop every other partition's column stats AssertHelpers.assertThrows( "Should reject partition-scoped column-stats ANALYZE for non-native partitioned tables", - IllegalArgumentException.class, expected, + IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS FOR COLUMNS") ); } @Test - public void testAnalyzeCatchesUpPartitionStats() { + public void testAnalyzeCatchesUpPartitionStats() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -462,6 +593,7 @@ public void testGetAggrColStatsForPartitioned() throws Exception { TableIdentifier identifier = TableIdentifier.of("default", "customers"); createPartitionedCustomers(identifier); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); HiveIcebergStorageHandler handler = storageHandler(); @@ -477,6 +609,44 @@ public void testGetAggrColStatsForPartitioned() throws Exception { // customer ids 0..2, one per last_name partition, merged across the three partitions Assert.assertEquals(0, statsObj.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(2, statsObj.getStatsData().getLongStats().getHighValue()); + + // single-partition probes pin each blob's content to its name, which the span check cannot + AggrStats brown = handler.getAggrColStatsFor(hmsTable, ImmutableList.of("customer_id"), + List.of("last_name=Brown")); + Assert.assertEquals(1, brown.getPartsFound()); + Assert.assertEquals(0, brown.getColStats().get(0).getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(0, brown.getColStats().get(0).getStatsData().getLongStats().getHighValue()); + AggrStats pink = handler.getAggrColStatsFor(hmsTable, ImmutableList.of("customer_id"), + List.of("last_name=Pink")); + Assert.assertEquals(1, pink.getPartsFound()); + Assert.assertEquals(2, pink.getColStats().get(0).getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2, pink.getColStats().get(0).getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testGetAggrColStatsForNullAndEmptyPartitions() throws Exception { + // NULL and empty-string partition values render as "last_name=null" and "last_name=" on both the + // blob-write side (ANALYZE) and the pruned-name side; a rendering mismatch silently drops the + // partition from the aggregation and partial aggregation extrapolates fabricated NDVs + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createPartitionedCustomers(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'Alice', NULL), (4, 'Eve', '')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + + List partNames = partitionNames(handler, hmsTable); + Assert.assertEquals(partNames.toString(), 5, partNames.size()); + Assert.assertTrue(partNames.toString(), partNames.contains("last_name=null")); + Assert.assertTrue(partNames.toString(), partNames.contains("last_name=")); + + // the blobs must carry the read side's names: an empty-string value that decodes as null would + // pass the aggregate below by double-serving the null partition's key + Assert.assertEquals(partNames.stream().sorted().toList(), colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "customer_id", partNames, 0, 4); } @Test @@ -505,6 +675,31 @@ public void testCountStarPartitioned() { } } + @Test + public void testBranchWriteComputesItsOwnPartitionStats() { + // the partition statistics file is anchored to a snapshot: a write to a branch computes it + // against the branch's head, leaving the table's own file and head where they were + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_part"); + createPartitionedCustomers(identifier); + + Table icebergTable = testTables.loadTable(identifier); + long tableSnapshotId = icebergTable.currentSnapshot().snapshotId(); + Assert.assertNotNull(IcebergTableUtil.getPartitionStatsFile(icebergTable, tableSnapshotId)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + icebergTable.refresh(); + Assert.assertEquals("the branch write leaves the table's head where it was", + tableSnapshotId, icebergTable.currentSnapshot().snapshotId()); + Assert.assertNotNull("the table keeps the partition statistics describing it", + IcebergTableUtil.getPartitionStatsFile(icebergTable, tableSnapshotId)); + Assert.assertNotNull("the branch's head carries the statistics its write computed", + IcebergTableUtil.getPartitionStatsFile(icebergTable, icebergTable.snapshot("b1").snapshotId())); + } + @Test public void testCountStarWithoutPartitionStatsFile() { assumeParquetHiveCatalogIceberg(); @@ -526,7 +721,7 @@ public void testCountStarWithoutPartitionStatsFile() { } @Test - public void testRowCountWithDeletes() { + public void testRowCountWithDeletes() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -572,21 +767,25 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { IcebergTableUtil.readPartitionStats(icebergTable, icebergTable.currentSnapshot()); Assert.assertEquals(3L, fileStats.get(DummyPartition.VOID).dataRecordCount().longValue()); - // column stats blobs are written for the physical partitions only: values existing solely among the - // legacy unpartitioned rows (Green, Pink) get no blob + // column stats blobs are written per physical partition; the legacy unpartitioned rows share one + // blob under the synthetic partition name, so values existing solely among them (Green, Pink) + // are accounted there List colStats = - IcebergTableUtil.readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId(), null); + IcebergTableUtil.readColStats( + icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); Assert.assertEquals( - List.of("last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), + List.of(DummyPartition.VOID, + "last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList()); + // the legacy blob covers exactly the rows written before the table was partitioned + ColumnStatisticsObj legacyCustomerId = colStatsObj(colStats, DummyPartition.VOID, "customer_id"); + Assert.assertEquals(0L, legacyCustomerId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, legacyCustomerId.getStatsData().getLongStats().getHighValue()); + // a blob describes the physical partition's files only: the legacy Brown row (customer_id 0) does not // merge into the Brown partition's blob, which covers just the new-spec row (customer_id 3) - ColumnStatisticsObj brownCustomerId = colStats.stream() - .filter(stats -> "last_name=Brown".equals(stats.getStatsDesc().getPartName())) - .flatMap(stats -> stats.getStatsObj().stream()) - .filter(obj -> "customer_id".equals(obj.getColName())) - .findFirst().orElseThrow(); + ColumnStatisticsObj brownCustomerId = colStatsObj(colStats, "last_name=Brown", "customer_id"); Assert.assertEquals(3L, brownCustomerId.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(3L, brownCustomerId.getStatsData().getLongStats().getHighValue()); @@ -620,7 +819,31 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { } @Test - public void testRowCountAfterEvolutionFromUnpartitioned() { + public void testAggrColStatsAfterEvolutionFromUnpartitioned() throws Exception { + // the legacy unpartitioned-spec rows are computed by a dedicated ANALYZE arm and stored under the + // synthetic partition's blob, so an aggregation over a pruned list holding the synthetic partition + // is complete: no extrapolation, and the legacy rows' values are accounted for + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createEvolvedCustomers(identifier); + // a genuine NULL partition value: its group's partition tuple is all null, exactly like the + // legacy rows' - only the spec id may tell them apart + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'Nia', NULL)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + List statNames = Lists.newArrayList(partitionNames(handler, hmsTable)); + statNames.add(DummyPartition.VOID); + Assert.assertTrue(statNames.toString(), statNames.contains("last_name=null")); + + // customer ids 0..2 exist only among the legacy unpartitioned rows, 3..6 in the partitioned ones + assertAggrColStatsRange(identifier, "customer_id", statNames, 0, 6); + } + + @Test + public void testRowCountAfterEvolutionFromUnpartitioned() throws SemanticException { assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -655,6 +878,620 @@ public void testRowCountAfterEvolutionFromUnpartitioned() { Assert.assertEquals(3L, brown.get(1)[0]); } + @Test + public void testAnalyzeColStatsInBatches() { + // guards the single-batch persist exemption: the storage handler holds one statistics file per + // snapshot, so honoring hive.stats.max.num.stats would let every batch replace the previous one + // (this cap would force one partition per batch and only the last would survive) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + createPartitionedCustomers(identifier); + // 3 stats objects per partition (customer_id, first_name, last_name): one partition per batch + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_MAX_NUM_STATS.varname, "3"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertEquals( + List.of("last_name=Brown", "last_name=Green", "last_name=Pink"), + colStatsPartNames(identifier)); + } + + @Test + public void testAggrColStatsAfterPartitionedSpecEvolution() { + // two partitioned specs: every row is grouped and its blob named under the spec that wrote it, + // in a single ANALYZE pass (the per-spec union rewrite could not even compile - the branches' + // partition structs had different field names) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "customers"); + PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) + .identity("last_name").build(); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, + fileFormat, ImmutableList.of(), formatVersion); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (first_name)"); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1, identifier, false)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertEquals( + List.of("first_name=Laci", "first_name=Marci", "first_name=Peti", + "last_name=Brown", "last_name=Green", "last_name=Pink"), + colStatsPartNames(identifier)); + } + + @Test + public void testAggrColStatsForYearTransformPartitions() throws Exception { + // time-transform partition values must render as Iceberg's human form ("2023"), not the raw + // transform ordinal ("53"): statistics and partition pruning join on the rendered name + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_by_year"); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), " + + "(2, date '2023-11-11'), (3, date '2024-06-01'), (4, date '1969-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // partition names as the read side renders them via partitionToPath (getPartitions cannot list + // non-identity transforms - its partition filter only supports identity columns); the year + // ordinal is negative for pre-1970 dates + List partNames = ImmutableList.of("d_year=1969", "d_year=2023", "d_year=2024"); + assertAggrColStatsRange(identifier, "id", partNames, 1, 4); + } + + @Test + public void testAggrColStatsForTimestampIdentityPartitions() throws Exception { + // identity-partitioned timestamps: Hive renders the value with a space separator, the blob name + // must carry Iceberg's ISO rendering (Conversions.fromPartitionString cannot parse timestamps) + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_by_ts"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, timestamp '2024-06-01 10:00:00'), (2, timestamp '2024-06-01 10:00:00'), " + + "(3, timestamp '2023-11-11 23:59:59')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // partitionToPath URL-escapes the ISO rendering (':' -> %3A) + List partNames = ImmutableList.of("ts=2023-11-11T23%3A59%3A59", "ts=2024-06-01T10%3A00%3A00"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAggrColStatsForCaseSensitivePartitionField() throws Exception { + // Hive's makePartName lowercases the wire keys, so the decode must match the partition field + // case-insensitively: a case-preserving field name (e.g. Spark-created) would otherwise decode + // every group's value to null + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_case"); + Schema schema = new Schema( + NestedField.optional(1, "id", Types.LongType.get()), + NestedField.optional(2, "eventDate", Types.DateType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("eventDate").build(); + testTables.createTable(shell, identifier.name(), schema, spec, fileFormat, ImmutableList.of(), formatVersion); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (2, date '2024-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = ImmutableList.of("eventDate=2023-03-04", "eventDate=2024-06-01"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 2); + } + + @Test + public void testAggrColStatsForTimestampLocalTZIdentityPartitions() throws Exception { + // identity-partitioned zoned timestamps: Hive renders the group value with a trailing zone id + // ("2024-06-01 10:00:00.0 UTC"), which the decode must map back to the instant's micros + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "events_by_ltz"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, ts timestamp with local time zone) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, timestamp '2024-06-01 10:00:00'), (2, timestamp '2024-06-01 10:00:00'), " + + "(3, timestamp '2023-11-11 23:59:59')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + List partNames = partitionNames(storageHandler(), hmsTable); + Assert.assertEquals(2, partNames.size()); + Assert.assertEquals(partNames.stream().sorted().toList(), colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAggrColStatsForTimeTransformEvolutions() throws Exception { + // year -> month -> day evolutions: one ANALYZE pass names each group's blob with the human + // rendering of the owning spec's transform ordinal + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_by_time"); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (month(d))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (day(d))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, date '2024-06-01')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = List.of("d_day=2024-06-01", "d_month=2023-11", "d_year=2023"); + Assert.assertEquals(partNames, colStatsPartNames(identifier)); + assertAggrColStatsRange(identifier, "id", partNames, 1, 3); + } + + @Test + public void testAutoGatherSkipsPartitionedInsert() { + // partition-level statistics are maintained by complete-scope writers only: with autogather + // on, a plain INSERT into a partitioned table persists nothing and the analyzed file carries + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_autogather"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11'), " + + "(3, date '2024-06-01')"); + // no new statistics file: the analyzed one keeps serving as an approximation + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023"), colStatsPartNames(identifier)); + } + + @Test + public void testTableLevelColStatsFallbackForPartitioned() throws Exception { + // hive.analyze.stmt.collect.partlevel.stats=false trades partition granularity for cheap + // maintenance: ANALYZE and autogather keep a single table-level file for the partitioned + // table, inserts merge into it incrementally, and planning serves it over the pruned set + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_tbl_level"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS.varname, false); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (3, date '2023-03-04')"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + // the file is table-level shaped: no blob carries a partition name + Assert.assertTrue(testTables.loadTable(identifier).statisticsFiles().stream() + .flatMap(statsFile -> statsFile.blobMetadata().stream()) + .noneMatch(blob -> blob.properties().containsKey(IcebergTableUtil.PARTITION_FIELD))); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, date '2024-05-05')"); + // the increment merged into the table-level statistics + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + + // the partition-level aggregation finds nothing to serve, at no read cost: the planner + // falls back to the table-level statistics + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable, ImmutableList.of("id"), + partitionNames(storageHandler(), hmsTable)); + Assert.assertEquals(0, aggrStats.getPartsFound()); + Assert.assertTrue(aggrStats.getColStats().isEmpty()); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse(colStatsAccurate(identifier)); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 3, 5); + } + + @Test + public void testAnalyzeReanchorsAfterStatsGap() { + // snapshots committed without statistics leave the stored file behind: the previous + // statistics keep serving as approximations, and ANALYZE re-anchors accounting every row + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_gap"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-11-11')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, date '2023-06-01')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + // the pre-gap statistics keep serving + Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List colStats = readCurrentColStats(identifier); + // after ANALYZE the statistics account for every row + ColumnStatisticsObj id2023 = colStatsObj(colStats, "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(3L, id2023.getStatsData().getLongStats().getHighValue()); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testColStatsServedButFrozenAfterDelete() { + // DML clears the accuracy flag: increments stop extending the statistics (ACID rule), the + // pre-delete file keeps serving as an approximation, and ANALYZE recomputes exactly + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_after_delete"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (2, date '2023-04-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + // the pre-delete statistics keep serving as an approximation + ColumnStatisticsObj id2023 = colStatsObj(readCurrentColStats(identifier), "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getHighValue()); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, date '2024-05-05')"); + // an insert maintains no partition-level statistics: the pre-delete file keeps serving + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023"), colStatsPartNames(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + Assert.assertEquals(List.of("d_year=2023", "d_year=2024"), colStatsPartNames(identifier)); + // recomputed: the deleted row no longer bounds the range + id2023 = colStatsObj(readCurrentColStats(identifier), "d_year=2023", "id"); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getHighValue()); + } + + + @Test + public void testColStatsNotAccurateAfterExternalWrite() { + // an engine that maintains no Hive statistics can commit at any time: the accuracy flag is + // trusted only while the current snapshot carries its own statistics file + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_external"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + // a foreign commit: new snapshot, no statistics file, no flag maintenance + testTables.loadTable(identifier).newAppend().commit(); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, date '2023-06-01')"); + // inserts maintain no partition-level statistics; the pre-existing ones keep serving as + // approximations until ANALYZE recomputes + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertFalse(colStatsAccurate(identifier)); + Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testSubsetColumnAnalyzeReplacesFile() { + // ANALYZE FOR COLUMNS on a subset replaces the statistics file whole: only the analyzed + // columns' statistics remain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_subset"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + createDatePartitionedTable(identifier); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (4, date '2023-04-04')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS id"); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj id2023 = colStatsObj(colStats, "d_year=2023", "id"); + Assert.assertEquals(1L, id2023.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(4L, id2023.getStatsData().getLongStats().getHighValue()); + Assert.assertTrue(colStats.stream() + .flatMap(stats -> stats.getStatsObj().stream()) + .noneMatch(obj -> "d".equals(obj.getColName()))); + } + + @Test + public void testAnalyzeRecomputesAfterDml() { + // DML stales the statistics; a table-wide ANALYZE recomputes every partition exactly and + // restores the accuracy flag + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_part_analyze"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 2"); + Assert.assertFalse(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List colStats = readCurrentColStats(identifier); + // p=a recomputed exactly: the deleted row no longer bounds it + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(1L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(1L, idA.getStatsData().getLongStats().getHighValue()); + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testColStatsSurviveDataNeutralRewrite() { + // a rewrite that changes no rows (compaction, Hive's or a foreign engine's) commits a + // "replace" snapshot: it neither outdates served statistics nor breaks the merge chain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_rewritten"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1)"); + Assert.assertTrue(colStatsAccurate(identifier)); + + commitDataNeutralRewrite(identifier); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2)"); + // the increment merged across the replace snapshot instead of being dropped + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 2); + } + + /** Rewrites the table's single data file into a byte-identical copy: a "replace" commit. */ + private void commitDataNeutralRewrite(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + try (CloseableIterable tasks = icebergTable.newScan().planFiles()) { + DataFile dataFile = tasks.iterator().next().file(); + String copyPath = dataFile.location() + "-copy"; + try (InputStream in = icebergTable.io().newInputFile(dataFile.location()).newStream(); + OutputStream out = icebergTable.io().newOutputFile(copyPath).create()) { + in.transferTo(out); + } + DataFile copy = DataFiles.builder(icebergTable.spec()) + .copy(dataFile) + .withPath(copyPath) + .build(); + icebergTable.newRewrite() + .rewriteFiles(Set.of(dataFile), Set.of(copy)) + .commit(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + Assert.assertEquals(DataOperations.REPLACE, + testTables.loadTable(identifier).currentSnapshot().operation()); + } + + @Test + public void testEmptyWriteKeepsColStats() { + // an insert that adds no files commits no snapshot: the statistics it computed describe + // nothing and must not replace the stored ones + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_empty_write"); + TableIdentifier source = TableIdentifier.of("default", "orders_empty_src"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("CREATE EXTERNAL TABLE " + source + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + long snapshotId = testTables.loadTable(identifier).currentSnapshot().snapshotId(); + + shell.executeStatement("INSERT INTO " + identifier + " SELECT id FROM " + source); + + // no commit, so the statistics of the unchanged snapshot keep serving, accurate + Assert.assertEquals(snapshotId, testTables.loadTable(identifier).currentSnapshot().snapshotId()); + checkColStatMinMaxValue(identifier.name(), "id", 1, 5); + Assert.assertTrue(colStatsAccurate(identifier)); + } + + @Test + public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { + // the same insert onto a table that carries no statistics: an increment gathered over no rows + // describes nothing, so it must not become the table's statistics either + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_empty_write_unanalyzed"); + TableIdentifier source = TableIdentifier.of("default", "orders_empty_write_src"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("CREATE EXTERNAL TABLE " + source + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("INSERT INTO " + identifier + " SELECT id FROM " + source); + + Assert.assertTrue("a write over no rows must not publish statistics", + readCurrentColStats(identifier).isEmpty()); + Assert.assertFalse(colStatsAccurate(identifier)); + } + + private boolean colStatsAccurate(TableIdentifier identifier) { + return StatsSetupConst.areColumnStatsUptoDate(hmsTable(identifier).getParameters(), "id"); + } + + private List readCurrentColStats(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + return IcebergTableUtil.readColStats( + icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); + } + + private boolean hasColStatsForCurrentSnapshot(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + long snapshotId = icebergTable.currentSnapshot().snapshotId(); + return icebergTable.statisticsFiles().stream().anyMatch(statsFile -> statsFile.snapshotId() == snapshotId); + } + + @Test + public void testIncrementalColStatsAfterTruncate() { + // truncate empties the table, so the next increment is the whole table and re-anchors the chain + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_truncated"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG " + + "STORED AS PARQUET TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (0), (1), (2)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 0, 2); + + shell.executeStatement("TRUNCATE TABLE " + identifier); + // the ancestor walk stops at the empty snapshot: nothing is served and no stats file is read + Assert.assertTrue(readCurrentColStats(identifier).isEmpty()); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (10), (11), (12)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 10, 12); + } + + @Test + public void testColStatsAfterPartitionTruncate() { + // a partition truncate clears the accuracy flag like any DML: the pre-truncate file keeps + // serving (the pruner never requests the wiped partition) and ANALYZE recomputes exactly + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_part_truncated"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + + shell.executeStatement("TRUNCATE TABLE " + identifier + " PARTITION (p = 'a')"); + Assert.assertFalse(colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + Assert.assertFalse(colStatsAccurate(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, idA.getStatsData().getLongStats().getHighValue()); + ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); + } + + + @Test + public void testVoidTransformEvolutionUnifiesPartitionNames() { + // a V1 removal keeps the field as a void transform: a legacy row with a null value and the + // new-spec rows project to the same unified partition tuple, so their statistics merge under + // one name - exactly as Iceberg's own partition statistics unify the rows + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_ambiguous"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, a string, b string) " + + "PARTITIONED BY SPEC (a, b) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='1')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'x', NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (a)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 'x', 'whatever')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj id = colStatsObj(colStats, "a=x/b=null", "id"); + Assert.assertEquals(1L, id.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2L, id.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testAggrColStatsAfterBucketAndYearEvolutionsFromUnpartitioned() throws Exception { + // unpartitioned history plus two partitioned specs - different bucket widths and a year + // transform - with null and empty-string source values scattered across all three: one ANALYZE + // pass groups every row under the spec that wrote it and names the blobs like the read side + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evolved"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, a string, b date) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (0, 'x', date '2023-03-04'), " + + "(1, '', NULL), (2, NULL, date '2024-06-01')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(8, a))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'x', date '2023-05-05'), " + + "(4, '', date '2023-06-06'), (5, NULL, NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(4, a), year(b))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'x', date '2023-07-07'), " + + "(7, '', date '2024-08-08'), (8, NULL, NULL)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // the null source values produce null partition values; the empty string hashes to a genuine bucket + List statNames = Stream.of( + DummyPartition.VOID, + "a_bucket_8=" + bucket(8, "x"), "a_bucket_8=" + bucket(8, ""), "a_bucket_8=null", + "a_bucket_4=" + bucket(4, "x") + "/b_year=2023", + "a_bucket_4=" + bucket(4, "") + "/b_year=2024", + "a_bucket_4=null/b_year=null") + .sorted().toList(); + + Assert.assertEquals(statNames, colStatsPartNames(identifier)); + // ids 0..2 exist only among the unpartitioned rows, 6..8 only in the latest spec's + assertAggrColStatsRange(identifier, "id", statNames, 0, 8); + } + + private static int bucket(int numBuckets, String value) { + return Transforms.bucket(numBuckets).bind(Types.StringType.get()).apply(value); + } + + private void createDatePartitionedTable(TableIdentifier identifier) { + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, d date) " + + "PARTITIONED BY SPEC (year(d)) STORED BY ICEBERG STORED AS PARQUET"); + } + + /** The named column's statistics object within the named partition's blob. */ + private static ColumnStatisticsObj colStatsObj(List colStats, String partName, String colName) { + return colStats.stream() + .filter(stats -> partName.equals(stats.getStatsDesc().getPartName())) + .flatMap(stats -> stats.getStatsObj().stream()) + .filter(obj -> colName.equals(obj.getColName())) + .findFirst().orElseThrow(); + } + + /** The persisted column-statistics blobs' partition names, sorted. */ + private List colStatsPartNames(TableIdentifier identifier) { + Table icebergTable = testTables.loadTable(identifier); + List colStats = + IcebergTableUtil.readColStats( + icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); + return colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList(); + } + + /** Asserts a complete aggregation over the given partition names: none missing, min/max spanning. */ + private void assertAggrColStatsRange(TableIdentifier identifier, String column, List statNames, + long lowValue, long highValue) throws Exception { + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable, ImmutableList.of(column), statNames); + Assert.assertEquals(statNames.size(), aggrStats.getPartsFound()); + ColumnStatisticsObj statsObj = aggrStats.getColStats().get(0); + Assert.assertEquals(lowValue, statsObj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(highValue, statsObj.getStatsData().getLongStats().getHighValue()); + } + private void createEvolvedCustomers(TableIdentifier identifier) { shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, @@ -693,6 +1530,58 @@ private void createPartitionedCustomers(TableIdentifier identifier, boolean auto HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); } + @Test + public void testPartitionNameRendersAcrossEvolutionsAndTypes() { + // SELECT PARTITION__NAME renders every row's name under its writing spec - identity with + // characters partitionToPath escapes, a day transform over timestamps, nulls, and the + // unpartitioned history - byte-equal to the names the partition listing produces + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_name_render"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, s string, ts timestamp) " + + "STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'x=1/y', timestamp '2023-03-04 10:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (s)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 'a b', timestamp '2023-03-04 11:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (day(ts))"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (3, 'c', timestamp '2024-06-01 12:00:00'), (4, NULL, NULL)"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (bucket(4, s))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'bucketed', timestamp '2025-01-01 00:00:00')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (truncate(2, s), month(ts))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (6, 'trunc-me', timestamp '2025-02-03 04:05:06')"); + + List rows = shell.executeStatement( + "SELECT id, PARTITION__NAME FROM " + identifier + " ORDER BY id"); + // every row renders its own writing spec's name + Assert.assertEquals(6, rows.size()); + Assert.assertEquals(DummyPartition.VOID, rows.get(0)[1]); + Assert.assertEquals("s=a+b", rows.get(1)[1]); + Assert.assertEquals("ts_day=2024-06-01", rows.get(2)[1]); + Assert.assertEquals("ts_day=null", rows.get(3)[1]); + Integer bucket = Transforms.bucket(4).bind(Types.StringType.get()).apply("bucketed"); + Assert.assertEquals("s_bucket_4=" + bucket, rows.get(4)[1]); + Assert.assertEquals("s_trunc_2=tr/ts_month=2025-02", rows.get(5)[1]); + Set served = rows.stream().map(r -> String.valueOf(r[1])).collect(Collectors.toSet()); + + // every spec's partitions, not only the latest spec's: rows keep their writing spec's name + Set expected; + try { + expected = storageHandler().getPartitions(hmsTable(identifier), Collections.emptyMap(), false).stream() + .map(Partition::getName) + .collect(Collectors.toSet()); + } catch (SemanticException e) { + throw new RuntimeException(e); + } + // the legacy unpartitioned rows belong to no partition: the synthetic no-partition name + expected.add(DummyPartition.VOID); + Assert.assertEquals(expected, served); + + // the analyzed statistics land under exactly the served names + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(expected.stream().sorted().toList(), colStatsPartNames(identifier)); + } + private org.apache.hadoop.hive.ql.metadata.Table hmsTable(TableIdentifier identifier) { try { return new org.apache.hadoop.hive.ql.metadata.Table( diff --git a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out index 23b2d41d3dd0..dfbdfaa45270 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out @@ -35,23 +35,23 @@ Stage-0 Stage-1 Map 1 vectorized File Output Operator [FS_23] - Map Join Operator [MAPJOIN_22] (rows=2 width=530) + Map Join Operator [MAPJOIN_22] (rows=2 width=607) BucketMapJoin:true,Conds:SEL_21._col1, _col2=RS_19._col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 2 [CUSTOM_EDGE] vectorized MULTICAST [RS_19] PartitionCols:_col2, _col1 - Select Operator [SEL_18] (rows=2 width=265) + Select Operator [SEL_18] (rows=2 width=552) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_17] (rows=2 width=265) + Filter Operator [FIL_17] (rows=2 width=552) predicate:(id is not null and part is not null) - TableScan [TS_3] (rows=2 width=265) - default@tbl,tbl2,Tbl:COMPLETE,Col:COMPLETE,Output:["foid","part","id"] - <-Select Operator [SEL_21] (rows=2 width=265) + TableScan [TS_3] (rows=2 width=552) + default@tbl,tbl2,Tbl:COMPLETE,Col:NONE,Output:["foid","part","id"] + <-Select Operator [SEL_21] (rows=2 width=552) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_20] (rows=2 width=265) + Filter Operator [FIL_20] (rows=2 width=552) predicate:(id is not null and part is not null) - TableScan [TS_0] (rows=2 width=265) - default@tbl,tbl,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] + TableScan [TS_0] (rows=2 width=552) + default@tbl,tbl,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] PREHOOK: query: SELECT * FROM tbl JOIN tbl tbl2 ON tbl.id = tbl2.id AND tbl.part = tbl2.part PREHOOK: type: QUERY diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out index ef5c4410fbb6..a5b35e5a76fe 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out @@ -359,7 +359,7 @@ num_trues num_falses bit_vector HL comment Transform: identity -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}} PREHOOK: query: EXPLAIN select count(*) from src_ice t1 join tbl_ice_puffin t2 on (t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@src_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index 4b4dd40489b9..a8b2550e29a1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -292,7 +292,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"c\",\"required\":false,\"type\":\"int\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index 97f2fb1f0303..1da70e101f39 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -121,77 +121,77 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(c) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Map 22 @@ -262,11 +262,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -277,13 +277,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 1407 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -294,23 +294,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -330,26 +330,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1547 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 773 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -359,13 +359,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 15 Reduce Operator Tree: @@ -376,11 +376,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 16 Reduce Operator Tree: @@ -391,13 +391,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 17 Reduce Operator Tree: @@ -408,16 +408,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 18 Execution mode: vectorized @@ -425,7 +425,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -444,20 +444,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 19 Execution mode: vectorized @@ -466,16 +466,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -486,11 +486,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized @@ -499,31 +499,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 21 Execution mode: vectorized @@ -532,11 +532,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 23 Execution mode: vectorized @@ -619,17 +619,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -640,11 +640,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: @@ -655,24 +655,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: @@ -683,17 +683,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized @@ -704,7 +704,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -719,13 +719,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Union 7 Vertex: Union 7 diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out index 0c539335f43f..5ba761a21709 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out @@ -296,20 +296,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) @@ -323,11 +323,11 @@ STAGE PLANS: Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (a <= 5) (type: boolean) @@ -383,24 +383,24 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: c (type: int) - minReductionHashAggr: 0.6666666 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -411,7 +411,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -430,13 +430,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1436 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1436 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -447,23 +447,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -483,26 +483,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -512,13 +512,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 14 Reduce Operator Tree: @@ -529,11 +529,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -544,13 +544,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 16 Reduce Operator Tree: @@ -561,16 +561,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 17 Execution mode: vectorized @@ -578,7 +578,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -597,17 +597,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 879 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -668,11 +668,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized @@ -707,31 +707,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 23 Execution mode: vectorized @@ -773,17 +773,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -794,11 +794,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: @@ -809,24 +809,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: @@ -837,14 +837,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -876,11 +876,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1380 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Union 7 Vertex: Union 7 @@ -988,58 +988,58 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 13 @@ -1071,20 +1071,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1104,26 +1104,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Execution mode: vectorized Reduce Operator Tree: @@ -1131,17 +1131,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -1152,11 +1152,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1167,24 +1167,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1195,14 +1195,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1217,13 +1217,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized @@ -1231,7 +1231,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1250,17 +1250,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1275,42 +1275,42 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reducer 9 Execution mode: vectorized Reduce Operator Tree: @@ -1318,11 +1318,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1426,58 +1426,58 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 13 @@ -1509,20 +1509,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1542,26 +1542,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Execution mode: vectorized Reduce Operator Tree: @@ -1569,11 +1569,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1584,11 +1584,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1599,24 +1599,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1627,14 +1627,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1649,13 +1649,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized @@ -1663,7 +1663,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1682,17 +1682,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1707,62 +1707,62 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col1 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reducer 9 Execution mode: vectorized Reduce Operator Tree: @@ -1770,17 +1770,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Union 5 Vertex: Union 5 diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out index d8f7ecf10e8d..6e11edce5a7e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out @@ -84,54 +84,54 @@ Stage-4 <-Reducer 2 [CONTAINS] File Output Operator [FS_46] table:{"name:":"default.ice01"} - Select Operator [SEL_44] (rows=7 width=206) + Select Operator [SEL_44] (rows=3 width=266) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_43] (rows=7 width=206) + Merge Join Operator [MERGEJOIN_43] (rows=3 width=266) Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_59] PartitionCols:_col4 - Select Operator [SEL_56] (rows=7 width=188) + Select Operator [SEL_56] (rows=6 width=280) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_53] (rows=7 width=188) + Filter Operator [FIL_53] (rows=6 width=76) predicate:((((id = 2) or (id > 4)) is null or (id < 2) or ((id > 2) and (id <= 4))) and FILE__PATH is not null) - TableScan [TS_0] (rows=7 width=188) - default@ice01,ice01,Tbl:COMPLETE,Col:NONE,Output:["id","name"] + TableScan [TS_0] (rows=7 width=78) + default@ice01,ice01,Tbl:COMPLETE,Col:COMPLETE,Output:["id","name"] <-Reducer 4 [SIMPLE_EDGE] vectorized SHUFFLE [RS_65] PartitionCols:_col0 - Group By Operator [GBY_64] (rows=3 width=4) + Group By Operator [GBY_64] (rows=3 width=184) Output:["_col0"],keys:_col0 - Select Operator [SEL_63] (rows=3 width=4) + Select Operator [SEL_63] (rows=3 width=184) Output:["_col0"] - Filter Operator [FIL_62] (rows=3 width=4) + Filter Operator [FIL_62] (rows=3 width=184) predicate:(row_number_window_0 = 1) - PTF Operator [PTF_61] (rows=7 width=4) + PTF Operator [PTF_61] (rows=6 width=184) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] - Select Operator [SEL_60] (rows=7 width=4) + Select Operator [SEL_60] (rows=6 width=184) Output:["_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_57] PartitionCols:FILE__PATH - Filter Operator [FIL_54] (rows=7 width=4) + Filter Operator [FIL_54] (rows=6 width=4) predicate:(((id = 2) or (id > 4)) and FILE__PATH is not null) Please refer to the previous TableScan [TS_0] <-Reducer 5 [CONTAINS] vectorized File Output Operator [FS_70] table:{"name:":"default.ice01"} - Select Operator [SEL_69] (rows=3 width=188) + Select Operator [SEL_69] (rows=3 width=266) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_68] (rows=3 width=188) + Filter Operator [FIL_68] (rows=3 width=258) predicate:(row_number_window_0 = 1) - PTF Operator [PTF_67] (rows=7 width=188) + PTF Operator [PTF_67] (rows=6 width=272) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] - Select Operator [SEL_66] (rows=7 width=188) + Select Operator [SEL_66] (rows=6 width=272) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_58] PartitionCols:FILE__PATH - Filter Operator [FIL_55] (rows=7 width=188) - predicate:((id = 2) or (id > 4)) + Filter Operator [FIL_55] (rows=6 width=76) + predicate:((id > 4) or (id = 2)) Please refer to the previous TableScan [TS_0] PREHOOK: query: delete from ice01 where id>4 OR id=2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out index e6227f4cae64..a8cc833ba713 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out @@ -306,17 +306,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -362,17 +362,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -381,10 +381,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -435,17 +435,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -475,17 +475,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -494,10 +494,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -566,17 +566,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -660,13 +660,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -675,17 +675,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -694,10 +694,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -757,17 +757,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -819,13 +819,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -834,17 +834,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -853,10 +853,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -922,17 +922,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -994,17 +994,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1013,10 +1013,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1067,17 +1067,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1107,17 +1107,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1126,10 +1126,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1191,17 +1191,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1247,17 +1247,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1266,10 +1266,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1320,17 +1320,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1360,17 +1360,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1379,10 +1379,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1442,17 +1442,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1501,17 +1501,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1520,10 +1520,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1574,17 +1574,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1617,17 +1617,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1636,10 +1636,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1690,17 +1690,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -1762,17 +1762,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1781,10 +1781,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1840,20 +1840,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -1937,13 +1937,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1952,17 +1952,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -1971,10 +1971,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2034,20 +2034,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -2131,13 +2131,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -2146,17 +2146,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -2165,10 +2165,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2223,48 +2223,48 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2275,17 +2275,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -2294,10 +2294,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2309,37 +2309,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -2348,37 +2348,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -2426,48 +2426,48 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2479,19 +2479,19 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reducer 3 Execution mode: vectorized Reduce Operator Tree: @@ -2499,10 +2499,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2514,37 +2514,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -2553,37 +2553,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -2633,56 +2633,56 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: ds (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 6 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2692,13 +2692,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 174892 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -2708,10 +2708,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2723,13 +2723,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 174892 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -2737,37 +2737,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Reducer 9 Execution mode: vectorized @@ -2776,37 +2776,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Union 3 Vertex: Union 3 @@ -2859,11 +2859,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -2872,17 +2872,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -2928,10 +2928,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2999,11 +2999,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3013,7 +3013,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3022,17 +3022,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3114,10 +3114,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3183,11 +3183,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3196,17 +3196,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3268,10 +3268,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3333,11 +3333,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3346,17 +3346,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3402,10 +3402,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3456,11 +3456,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3469,17 +3469,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3528,10 +3528,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3599,14 +3599,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3616,7 +3616,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3625,17 +3625,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3717,10 +3717,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3779,14 +3779,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3796,7 +3796,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3805,17 +3805,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3897,10 +3897,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3955,11 +3955,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 @@ -3969,50 +3969,50 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Union 5 - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 3 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -4022,10 +4022,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4037,37 +4037,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Reducer 6 Execution mode: vectorized @@ -4076,37 +4076,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -4181,11 +4181,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -4194,17 +4194,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2000 Data size: 16000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Map 3 Map Operator Tree: @@ -4263,10 +4263,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out index cd09db327734..eedb29f7095a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out @@ -132,7 +132,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -148,7 +148,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -156,15 +156,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -226,7 +225,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -238,21 +237,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -315,7 +313,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -331,7 +329,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col2","_col1","iceberg_bucket(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1, iceberg_bucket(_col2, 3) Select Operator [SEL_12] (rows=22 width=94) Output:["_col0","_col1","_col2"] @@ -339,15 +337,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=11 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_19] (rows=11 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=11 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + Select Operator [SEL_20] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_19] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_14] (rows=22 width=94) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_12] @@ -450,7 +447,7 @@ POSTHOOK: Output: default@tbl_target_mixed Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -462,11 +459,11 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=3 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=3 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_19] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_14] table:{"name:":"default.tbl_target_mixed"} Select Operator [SEL_13] (rows=4 width=99) @@ -475,10 +472,9 @@ Stage-3 predicate:(b = 'EUR') TableScan [TS_0] (rows=22 width=94) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=3 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_15] (rows=4 width=99) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_13] @@ -502,7 +498,7 @@ POSTHOOK: Output: default@tbl_target_mixed Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -514,11 +510,11 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=1030) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=591) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_19] (rows=1 width=794) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=500) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_14] table:{"name:":"default.tbl_target_mixed"} Select Operator [SEL_13] (rows=1 width=99) @@ -527,10 +523,9 @@ Stage-3 predicate:((c = 100L) and (b = 'USD')) TableScan [TS_0] (rows=22 width=94) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=1 width=659) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"],keys:ccy, iceberg_bucket(c, 3) + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=568) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"] Select Operator [SEL_15] (rows=1 width=99) Output:["a","ccy","c"] Please refer to the previous Select Operator [SEL_13] @@ -677,7 +672,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -693,7 +688,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col1, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -701,15 +696,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=5 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=5 width=516) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=5 width=584) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(ccy, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -772,7 +766,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -788,7 +782,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col0, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col0, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -796,15 +790,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=21 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=21 width=516) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=21 width=584) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(id)","max(id)","count(1)","count(id)","compute_bit_vector_hll(id)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(id, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(id)","max(id)","count(1)","count(id)","compute_bit_vector_hll(id)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["ccy","id"] Please refer to the previous Select Operator [SEL_12] @@ -867,7 +860,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -883,7 +876,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_truncate(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_truncate(_col1, 2) Select Operator [SEL_12] (rows=22 width=11) Output:["_col0","_col1"] @@ -891,15 +884,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","c"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=8 width=752) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=8 width=520) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=8 width=520) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(ccy, 2) + Select Operator [SEL_20] (rows=1 width=528) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=336) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=336) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=11) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -962,7 +954,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -978,7 +970,7 @@ Stage-3 Select Operator [SEL_18] Output:["_col0","_col1","_col2","iceberg_truncate(_col1, 2)","iceberg_truncate(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_14] + PARTITION_ONLY_SHUFFLE [RS_14] PartitionCols:iceberg_truncate(_col1, 2), iceberg_truncate(_col2, 3) Select Operator [SEL_13] (rows=22 width=199) Output:["_col0","_col1","_col2"] @@ -986,15 +978,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_22] - Select Operator [SEL_21] (rows=5 width=1421) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_20] (rows=5 width=1076) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=5 width=1144) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(b))","avg(COALESCE(length(b),0))","count(b)","compute_bit_vector_hll(b)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(b, 2), iceberg_truncate(ccy, 3) + Select Operator [SEL_21] (rows=1 width=1005) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_20] (rows=1 width=708) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_17] + Group By Operator [GBY_16] (rows=1 width=776) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(b))","avg(COALESCE(length(b),0))","count(b)","compute_bit_vector_hll(b)","min(ccy)","max(ccy)","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_15] (rows=22 width=199) Output:["a","b","ccy"] Please refer to the previous Select Operator [SEL_13] @@ -1063,7 +1054,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1079,7 +1070,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_year(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_year(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1091,15 +1082,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_year(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1149,7 +1139,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1165,7 +1155,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_year(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_year(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1177,15 +1167,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_year(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1235,7 +1224,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1251,7 +1240,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_month(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_month(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1263,15 +1252,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_month(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1321,7 +1309,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1337,7 +1325,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_month(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_month(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1349,15 +1337,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_month(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1407,7 +1394,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1423,7 +1410,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_day(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_day(_col1) Select Operator [SEL_3] (rows=1 width=240) Output:["_col0","_col1","_col2"] @@ -1435,15 +1422,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=946) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=604) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=672) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_day(date_time_date) + Select Operator [SEL_19] (rows=1 width=890) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=596) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=664) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=240) Output:["id","year_partition","date_time_date"] Please refer to the previous Select Operator [SEL_3] @@ -1493,7 +1479,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1509,7 +1495,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_day(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_day(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1521,15 +1507,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_day(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1579,7 +1564,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1595,7 +1580,7 @@ Stage-3 Select Operator [SEL_16] Output:["_col0","_col1","_col2","_col2","iceberg_hour(_col1)"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col2, iceberg_hour(_col1) Select Operator [SEL_3] (rows=1 width=224) Output:["_col0","_col1","_col2"] @@ -1607,15 +1592,14 @@ Stage-3 _dummy_database@_dummy_table,_dummy_table,Tbl:COMPLETE,Col:COMPLETE Reducer 3 vectorized File Output Operator [FS_20] - Select Operator [SEL_19] (rows=1 width=919) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_18] (rows=1 width=572) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=1 width=640) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],keys:year_partition, iceberg_hour(date_time_timestamp) + Select Operator [SEL_19] (rows=1 width=863) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_18] (rows=1 width=564) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_9] + Group By Operator [GBY_8] (rows=1 width=632) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"] Select Operator [SEL_7] (rows=1 width=224) Output:["id","year_partition","date_time_timestamp"] Please refer to the previous Select Operator [SEL_3] @@ -1650,7 +1634,7 @@ POSTHOOK: Output: default@tbl_target_identity Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1662,21 +1646,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_identity"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1692,7 +1675,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1704,21 +1687,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1735,7 +1717,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1751,7 +1733,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1759,15 +1741,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1784,7 +1765,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1800,7 +1781,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 3) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1808,15 +1789,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1833,7 +1813,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1849,7 +1829,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1857,15 +1837,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1882,7 +1861,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1898,7 +1877,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 3) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -1906,15 +1885,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] @@ -1930,7 +1908,7 @@ POSTHOOK: Output: default@tbl_target_identity Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1942,21 +1920,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=6 width=754) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=6 width=419) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_identity"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=6 width=487) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:ccy + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -1972,7 +1949,7 @@ POSTHOOK: Output: default@tbl_target_bucket Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -1984,21 +1961,20 @@ Stage-3 Stage-1 Reducer 2 vectorized File Output Operator [FS_17] - Select Operator [SEL_16] (rows=3 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_15] (rows=3 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized + Select Operator [SEL_16] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_15] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized File Output Operator [FS_11] table:{"name:":"default.tbl_target_bucket"} Select Operator [SEL_10] (rows=22 width=87) Output:["_col0","_col1"] TableScan [TS_0] (rows=22 width=87) default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=4 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 3) + PARTITION_ONLY_SHUFFLE [RS_14] + Group By Operator [GBY_13] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_12] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_10] @@ -2031,7 +2007,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -2047,7 +2023,7 @@ Stage-3 Select Operator [SEL_17] Output:["_col0","_col1","iceberg_bucket(_col1, 2)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_13] + PARTITION_ONLY_SHUFFLE [RS_13] PartitionCols:iceberg_bucket(_col1, 2) Select Operator [SEL_12] (rows=22 width=87) Output:["_col0","_col1"] @@ -2055,15 +2031,14 @@ Stage-3 default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] Reducer 3 vectorized File Output Operator [FS_21] - Select Operator [SEL_20] (rows=2 width=574) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Group By Operator [GBY_19] (rows=2 width=336) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=3 width=404) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_bucket(ccy, 2) + Select Operator [SEL_20] (rows=1 width=530) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] + Group By Operator [GBY_19] (rows=1 width=332) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=1 width=400) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"] Select Operator [SEL_14] (rows=22 width=87) Output:["a","ccy"] Please refer to the previous Select Operator [SEL_12] diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out index 0ace027030fa..36b9ab3d4dff 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out @@ -80,54 +80,54 @@ STAGE PLANS: TableScan alias: srcpart_date_n7 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_n3_key1_min) AND DynamicValue(RS_7_srcpart_small_n3_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_n3_key1_bloom_filter))) (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_small_n3 filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20) - minReductionHashAggr: 0.95 + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=19) + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized Reducer 2 @@ -138,17 +138,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 126 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2090 Data size: 365523 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -157,10 +157,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -169,14 +169,14 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=20) + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=19) mode: final outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index e1e5d28cf61b..a474fab5216e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -67,34 +67,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -150,7 +146,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -184,34 +180,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -267,7 +259,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -275,14 +267,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: strcol (type: string), intcol (type: int), 3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -291,38 +283,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -378,7 +366,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -386,14 +374,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: strcol (type: string), intcol (type: int), 4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -402,38 +390,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -495,7 +479,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"intcol\":\"true\",\"pcol\":\"true\",\"strcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"strcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"int\"}]} @@ -572,7 +556,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -606,34 +590,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -690,7 +670,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -724,34 +704,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -808,7 +784,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -842,34 +818,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -941,43 +913,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -988,7 +957,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -999,17 +968,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1064,43 +1032,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 12 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1111,7 +1076,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1122,17 +1087,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1187,43 +1151,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 24 Data size: 4440 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9583333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1234,7 +1195,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1245,17 +1206,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1310,43 +1270,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 48 Data size: 8880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9791667 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1357,7 +1314,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1368,17 +1325,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1435,43 +1391,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 96 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9895833 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1482,7 +1435,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1493,17 +1446,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1560,43 +1512,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 192 Data size: 34560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1607,7 +1556,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1618,17 +1567,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1692,7 +1640,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -2153,7 +2101,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2187,34 +2135,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2270,7 +2214,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2304,34 +2248,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2387,7 +2327,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2421,34 +2361,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2518,21 +2454,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2541,38 +2477,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2628,21 +2560,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2651,38 +2583,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2746,7 +2674,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -2847,7 +2775,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2881,34 +2809,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2964,7 +2888,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2998,34 +2922,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3081,7 +3001,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3115,34 +3035,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3212,21 +3128,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 45637829068876994L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3235,38 +3151,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3322,21 +3234,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3345,38 +3257,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3438,7 +3346,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"long\"}]} @@ -3539,7 +3447,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3573,34 +3481,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3656,7 +3560,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3690,34 +3594,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3773,7 +3673,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3807,34 +3707,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3904,21 +3800,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3927,38 +3823,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4014,21 +3906,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -4037,38 +3929,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4130,7 +4018,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"double\"}]} @@ -4231,7 +4119,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4265,34 +4153,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4348,7 +4232,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4382,34 +4266,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4465,7 +4345,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4499,34 +4379,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4596,7 +4472,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4622,35 +4498,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 180 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4706,7 +4578,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4732,35 +4604,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4822,7 +4690,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(10, 6)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index a0ad059f0a83..fc32aedc3ac5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -69,34 +69,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -152,7 +148,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -186,34 +182,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -269,7 +261,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -303,34 +295,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -400,7 +388,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -426,35 +414,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -510,7 +494,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -536,35 +520,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -626,7 +606,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -729,7 +709,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -763,34 +743,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -846,7 +822,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -880,34 +856,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -963,7 +935,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -997,34 +969,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1094,7 +1062,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1120,35 +1088,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1204,7 +1168,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1230,35 +1194,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1320,7 +1280,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -1423,7 +1383,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1457,34 +1417,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1540,7 +1496,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1574,34 +1530,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1657,7 +1609,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1691,34 +1643,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1788,7 +1736,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1814,35 +1762,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1898,7 +1842,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1924,35 +1868,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2014,7 +1954,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2117,7 +2057,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2151,34 +2091,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2234,7 +2170,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2268,34 +2204,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2351,21 +2283,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_truncate - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'gfhuiyoprj' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2374,38 +2306,34 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2467,7 +2395,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2554,7 +2482,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2588,34 +2516,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2671,7 +2595,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2705,34 +2629,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2788,21 +2708,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_bucket - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'gfhutjkgkd' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2811,38 +2731,34 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_bucket(pcol, 16) (type: int) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2904,7 +2820,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2992,7 +2908,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3025,18 +2941,15 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_bucket(pcol, 16) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(38,0)), _col2 (type: decimal(38,0)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(38,0)), _col1 (type: decimal(38,0)), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -3057,17 +2970,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DECIMAL' (type: string), _col1 (type: decimal(38,0)), _col2 (type: decimal(38,0)), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), named_struct('pcol_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DECIMAL' (type: string), _col0 (type: decimal(38,0)), _col1 (type: decimal(38,0)), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3129,7 +3041,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(38, 0)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 55905a907b95..66c45229016a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -56,7 +56,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -90,34 +90,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), max(length(b)), avg(COALESCE(length(b),0)), count(b), compute_bit_vector_hll(b) - keys: iceberg_truncate(b, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: struct), _col8 (type: bigint), _col9 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), UDFToLong(_col1) (type: bigint), UDFToLong(_col2) (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint), COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), named_struct('b_trunc_2',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 754 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), UDFToLong(_col0) (type: bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint), COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 754 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -178,7 +174,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 9212bb8dc3c8..2692b6e0d623 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -57,14 +57,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -73,38 +73,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -160,7 +156,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -168,14 +164,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -184,38 +180,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -277,7 +269,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"intcol\":\"true\",\"pcol\":\"true\",\"strcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"strcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"int\"}]} @@ -358,7 +350,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -392,34 +384,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 173 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -497,43 +485,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -544,7 +529,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -555,17 +540,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -620,43 +604,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -667,7 +648,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -678,17 +659,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -743,43 +723,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -790,7 +767,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -801,17 +778,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -866,43 +842,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1037 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -913,7 +886,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -924,17 +897,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 833 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -991,43 +963,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1038,7 +1007,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1049,17 +1018,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1116,43 +1084,40 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - keys: country (type: string), state (type: string) - minReductionHashAggr: 0.95 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 1039 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int), _col3 (type: struct), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: int), _col9 (type: bigint), _col10 (type: binary), _col11 (type: int), _col12 (type: struct), _col13 (type: bigint), _col14 (type: binary), _col15 (type: int), _col16 (type: struct), _col17 (type: bigint), _col18 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -1163,7 +1128,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1174,17 +1139,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) - keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 835 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'LONG' (type: string), UDFToLong(_col7) (type: bigint), UDFToLong(_col8) (type: bigint), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col11,0)) (type: bigint), COALESCE(_col12,0) (type: double), (_col4 - _col13) (type: bigint), COALESCE(ndv_compute_bit_vector(_col14),0) (type: bigint), _col14 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col15,0)) (type: bigint), COALESCE(_col16,0) (type: double), (_col4 - _col17) (type: bigint), COALESCE(ndv_compute_bit_vector(_col18),0) (type: bigint), _col18 (type: binary), named_struct('country',_col0,'state',_col1) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1478 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1248,7 +1212,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -1371,7 +1335,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1405,34 +1369,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1500,21 +1460,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-31' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1523,38 +1483,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1610,21 +1566,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-26' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1633,38 +1589,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: date) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: date) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: date) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 656 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1726,7 +1678,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -1837,7 +1789,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1871,34 +1823,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1966,21 +1914,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 34567890123456787L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1989,38 +1937,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2076,21 +2020,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2099,38 +2043,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: bigint) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), _col10 (type: bigint), _col11 (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2192,7 +2132,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"long\"}]} @@ -2303,7 +2243,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2337,34 +2277,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2432,21 +2368,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2455,38 +2391,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2542,21 +2474,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2565,38 +2497,34 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: double) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: double) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DOUBLE' (type: string), _col10 (type: double), _col11 (type: double), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2658,7 +2586,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"double\"}]} @@ -2769,7 +2697,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2803,34 +2731,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1251 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2898,7 +2822,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2924,35 +2848,31 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3008,7 +2928,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3034,35 +2954,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: pcol (type: decimal(10,6)) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: decimal(10,6)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(10,6)) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: decimal(10,6)) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DECIMAL' (type: string), _col10 (type: decimal(10,6)), _col11 (type: decimal(10,6)), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3124,7 +3040,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(10, 6)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 547b1e0d3d68..9f90a4898fc8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -69,34 +69,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -152,7 +148,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -186,34 +182,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -269,7 +261,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -303,34 +295,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -396,7 +384,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -422,35 +410,31 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -506,7 +490,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -532,35 +516,31 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - keys: iceberg_year(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: int), _col7 (type: int), _col8 (type: bigint), _col9 (type: binary), _col10 (type: date), _col11 (type: date), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), UDFToLong(_col6) (type: bigint), UDFToLong(_col7) (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'DATE' (type: string), _col10 (type: date), _col11 (type: date), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_year',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -622,7 +602,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -703,7 +683,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -737,34 +717,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -820,7 +796,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -854,34 +830,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -937,7 +909,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -971,34 +943,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1064,7 +1032,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1090,35 +1058,31 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1174,7 +1138,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1200,35 +1164,31 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_month(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: date), _col7 (type: date), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'LONG' (type: string), _col1 (type: bigint), _col2 (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DATE' (type: string), _col6 (type: date), _col7 (type: date), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_month',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1290,7 +1250,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -1371,7 +1331,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1405,34 +1365,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1488,7 +1444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1522,34 +1478,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1605,7 +1557,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1639,34 +1591,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1736,7 +1684,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1762,35 +1710,31 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1846,7 +1790,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1872,35 +1816,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_day(pcol) (type: int) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'DATE' (type: string), _col1 (type: date), _col2 (type: date), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_day',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1962,7 +1902,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2065,7 +2005,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2099,34 +2039,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2182,7 +2118,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2216,34 +2152,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2299,21 +2231,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: ice_parquet_date_transform_truncate - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'gfhuiyoprj' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2322,38 +2254,34 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - keys: iceberg_truncate(pcol, 2) (type: string) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: struct), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 684 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col1,0)) (type: bigint), COALESCE(_col2,0) (type: double), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'LONG' (type: string), _col6 (type: bigint), _col7 (type: bigint), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('pcol_trunc',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1018 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2415,7 +2343,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out index bba70b438c06..f24d9836fbaf 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out @@ -24,7 +24,7 @@ POSTHOOK: Input: default@ice_parquet__temp_table_for_load_data__ POSTHOOK: Output: default@ice_parquet Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -40,7 +40,7 @@ Stage-3 Select Operator [SEL_19] Output:["_col0","_col1","_col2","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_15] + PARTITION_ONLY_SHUFFLE [RS_15] PartitionCols:_col2 Select Operator [SEL_14] (rows=77 width=187) Output:["_col0","_col1","_col2"] @@ -48,15 +48,14 @@ Stage-3 default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"] Reducer 3 vectorized File Output Operator [FS_23] - Select Operator [SEL_22] (rows=38 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_21] (rows=38 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=77 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol + Select Operator [SEL_22] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_21] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_18] + Group By Operator [GBY_17] (rows=1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"] Select Operator [SEL_16] (rows=77 width=187) Output:["strcol","intcol","pcol"] Please refer to the previous Select Operator [SEL_14] @@ -79,7 +78,7 @@ POSTHOOK: Input: default@ice_parquet__temp_table_for_load_data__ POSTHOOK: Output: default@ice_parquet Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -95,7 +94,7 @@ Stage-3 Select Operator [SEL_19] Output:["_col0","_col1","_col2","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_15] + PARTITION_ONLY_SHUFFLE [RS_15] PartitionCols:_col2 Select Operator [SEL_14] (rows=77/6 width=187) Output:["_col0","_col1","_col2"] @@ -103,15 +102,14 @@ Stage-3 default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"] Reducer 3 vectorized File Output Operator [FS_23] - Select Operator [SEL_22] (rows=38/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"] - Group By Operator [GBY_21] (rows=38/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=77/3 width=187) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol + Select Operator [SEL_22] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] + Group By Operator [GBY_21] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_18] + Group By Operator [GBY_17] (rows=1/1 width=752) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"] Select Operator [SEL_16] (rows=77/6 width=187) Output:["strcol","intcol","pcol"] Please refer to the previous Select Operator [SEL_14] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out index fae1fb8c26df..17e9f416d971 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -58,14 +58,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is null (type: boolean) - Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -109,14 +109,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out index c6365d049e94..22498fe40b83 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out @@ -54,14 +54,14 @@ STAGE PLANS: TableScan alias: ice01 filterExpr: ((year = 2023) and (month = 10) and (day = 3)) (type: boolean) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index d6a09b1d53c6..de26ccd32be0 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -85,17 +85,17 @@ STAGE PLANS: TableScan alias: test_ice_int filterExpr: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -280,17 +280,17 @@ STAGE PLANS: TableScan alias: test_ice_bigint filterExpr: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 226784902765739L (type: bigint), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -501,17 +501,17 @@ STAGE PLANS: TableScan alias: test_ice_str filterExpr: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), 'ddd' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -930,17 +930,17 @@ STAGE PLANS: TableScan alias: test_ice_double filterExpr: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), b (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index 3e3f7761f053..ae9c6aa6a61f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -107,7 +107,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 format-version 2 @@ -362,7 +362,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 format-version 2 @@ -434,7 +434,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 format-version 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out index 7ce866f2b03c..ecf1ece52e32 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out @@ -252,36 +252,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=798) + Select Operator [SEL_37] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=500) + Group By Operator [GBY_36] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=785 width=366) + Select Operator [SEL_31] (rows=1045 width=178) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=785 width=186) + Map Join Operator [MAPJOIN_30] (rows=1045 width=178) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=238 width=95) + Select Operator [SEL_26] (rows=227 width=180) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=238 width=95) + Filter Operator [FIL_25] (rows=227 width=180) predicate:key is not null - TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=1000 width=95) + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Select Operator [SEL_29] (rows=950 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=1000 width=95) + Filter Operator [FIL_28] (rows=950 width=178) predicate:key is not null - TableScan [TS_3] (rows=1000 width=95) - default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=1000 width=178) + default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=785 width=366) + Select Operator [SEL_33] (rows=1045 width=178) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] @@ -416,36 +416,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=798) + Select Operator [SEL_37] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=500) + Group By Operator [GBY_36] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=809 width=366) + Select Operator [SEL_31] (rows=547 width=179) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=809 width=186) + Map Join Operator [MAPJOIN_30] (rows=547 width=179) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=238 width=95) + Select Operator [SEL_26] (rows=227 width=180) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=238 width=95) + Filter Operator [FIL_25] (rows=227 width=180) predicate:key is not null - TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=524 width=95) + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Select Operator [SEL_29] (rows=498 width=179) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=524 width=95) + Filter Operator [FIL_28] (rows=498 width=179) predicate:key is not null - TableScan [TS_3] (rows=524 width=95) - default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=524 width=179) + default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=809 width=366) + Select Operator [SEL_33] (rows=547 width=179) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out index f21a3c84da24..a344f7479452 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out @@ -129,23 +129,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=372 width=8) + Map Join Operator [MAPJOIN_26] (rows=5 width=198) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=238 width=4) + Select Operator [SEL_22] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_21] (rows=238 width=89) + Filter Operator [FIL_21] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=238 width=4) + TableScan [TS_3] (rows=238 width=180) + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + <-Select Operator [SEL_25] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_24] (rows=238 width=89) + Filter Operator [FIL_24] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b @@ -203,23 +203,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_33] Group By Operator [GBY_32] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_31] (rows=372 width=8) + Map Join Operator [MAPJOIN_31] (rows=5 width=198) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=238 width=4) + Select Operator [SEL_27] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_26] (rows=238 width=89) + Filter Operator [FIL_26] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - <-Select Operator [SEL_30] (rows=238 width=4) + TableScan [TS_3] (rows=238 width=180) + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + <-Select Operator [SEL_30] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_29] (rows=238 width=89) + Filter Operator [FIL_29] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out index dd93fcf31851..857c74642a6f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out @@ -242,23 +242,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=1797 width=8) + Map Join Operator [MAPJOIN_26] (rows=732 width=179) Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=738 width=4) + Select Operator [SEL_22] (rows=666 width=179) Output:["_col0"] - Filter Operator [FIL_21] (rows=738 width=89) - predicate:(part is not null and key is not null) - TableScan [TS_3] (rows=738 width=89) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=738 width=4) + Filter Operator [FIL_21] (rows=666 width=179) + predicate:(key is not null and part is not null) + TableScan [TS_3] (rows=738 width=179) + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + <-Select Operator [SEL_25] (rows=666 width=179) Output:["_col0"] - Filter Operator [FIL_24] (rows=738 width=89) - predicate:(part is not null and key is not null) - TableScan [TS_0] (rows=738 width=89) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + Filter Operator [FIL_24] (rows=666 width=179) + predicate:(key is not null and part is not null) + TableScan [TS_0] (rows=738 width=179) + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:NONE,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b @@ -309,29 +309,29 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_37] Group By Operator [GBY_36] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_35] (rows=1797 width=8) + Map Join Operator [MAPJOIN_35] (rows=732 width=179) Conds:SEL_34._col0, _col1=RS_29._col0, _col1(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_29] PartitionCols:_col0, _col1 - Select Operator [SEL_28] (rows=738 width=89) + Select Operator [SEL_28] (rows=666 width=179) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=738 width=89) - predicate:(part is not null and key is not null) - TableScan [TS_3] (rows=738 width=89) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=85) - Group By Operator [GBY_31] (rows=2 width=85) + Filter Operator [FIL_27] (rows=666 width=179) + predicate:(key is not null and part is not null) + TableScan [TS_3] (rows=738 width=179) + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + Dynamic Partitioning Event Operator [EVENT_32] (rows=666 width=179) + Group By Operator [GBY_31] (rows=666 width=179) Output:["_col0"],keys:_col0 - Select Operator [SEL_30] (rows=738 width=85) + Select Operator [SEL_30] (rows=666 width=179) Output:["_col0"] Please refer to the previous Select Operator [SEL_28] - <-Select Operator [SEL_34] (rows=738 width=89) + <-Select Operator [SEL_34] (rows=666 width=179) Output:["_col0","_col1"] - Filter Operator [FIL_33] (rows=738 width=89) - predicate:(part is not null and key is not null) - TableScan [TS_0] (rows=738 width=89) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] + Filter Operator [FIL_33] (rows=666 width=179) + predicate:(key is not null and part is not null) + TableScan [TS_0] (rows=738 width=179) + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:NONE,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out index 8ff5e300aa46..25a47b5d3522 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out @@ -181,23 +181,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_33] Group By Operator [GBY_32] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_31] (rows=372 width=8) + Map Join Operator [MAPJOIN_31] (rows=5 width=198) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=238 width=4) + Select Operator [SEL_27] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_26] (rows=238 width=89) + Filter Operator [FIL_26] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - <-Select Operator [SEL_30] (rows=238 width=4) + TableScan [TS_3] (rows=238 width=180) + default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + <-Select Operator [SEL_30] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_29] (rows=238 width=89) + Filter Operator [FIL_29] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b @@ -248,23 +248,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=372 width=8) + Map Join Operator [MAPJOIN_26] (rows=5 width=198) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=238 width=4) + Select Operator [SEL_22] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_21] (rows=238 width=89) + Filter Operator [FIL_21] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=89) - default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=238 width=4) + TableScan [TS_3] (rows=238 width=180) + default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + <-Select Operator [SEL_25] (rows=5 width=180) Output:["_col0"] - Filter Operator [FIL_24] (rows=238 width=89) + Filter Operator [FIL_24] (rows=5 width=180) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=89) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out index 024fde7f1576..afde412850de 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out @@ -142,36 +142,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=798) + Select Operator [SEL_37] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=500) + Group By Operator [GBY_36] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result_n3"} - Select Operator [SEL_31] (rows=809 width=366) + Select Operator [SEL_31] (rows=547 width=179) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=809 width=186) + Map Join Operator [MAPJOIN_30] (rows=547 width=179) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=238 width=95) + Select Operator [SEL_26] (rows=227 width=180) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=238 width=95) + Filter Operator [FIL_25] (rows=227 width=180) predicate:key is not null - TableScan [TS_0] (rows=238 width=95) - default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=524 width=95) + TableScan [TS_0] (rows=238 width=180) + default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Select Operator [SEL_29] (rows=498 width=179) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=524 width=95) + Filter Operator [FIL_28] (rows=498 width=179) predicate:key is not null - TableScan [TS_3] (rows=524 width=95) - default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=524 width=179) + default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=809 width=366) + Select Operator [SEL_33] (rows=547 width=179) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out index 7b486db67697..b6244c744e3d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out @@ -41,40 +41,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (CUSTOM_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_46] - Limit [LIM_45] (rows=20 width=447) + Limit [LIM_45] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_44] (rows=791 width=447) + Select Operator [SEL_44] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_43] - Top N Key Operator [TNK_42] (rows=791 width=447) + Top N Key Operator [TNK_42] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_41] (rows=791 width=447) - BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [CUSTOM_EDGE] vectorized, llap - MULTICAST [RS_38] + Map Join Operator [MAPJOIN_41] (rows=550 width=178) + Conds:RS_38._col0, _col1=SEL_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_38] PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_36] (rows=500 width=178) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_40] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=500 width=269) - predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + Select Operator [SEL_37] (rows=450 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_36] (rows=450 width=525) + predicate:(key1 is not null and key2 is not null) + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_40] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_39] (rows=500 width=178) + predicate:(key is not null and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -139,40 +139,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (CUSTOM_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_46] - Limit [LIM_45] (rows=20 width=447) + Limit [LIM_45] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_44] (rows=791 width=447) + Select Operator [SEL_44] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_43] - Top N Key Operator [TNK_42] (rows=791 width=447) + Top N Key Operator [TNK_42] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_41] (rows=791 width=447) - BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [CUSTOM_EDGE] vectorized, llap - MULTICAST [RS_38] + Map Join Operator [MAPJOIN_41] (rows=550 width=178) + Conds:RS_38._col0, _col1=SEL_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_38] PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_36] (rows=500 width=178) - predicate:((key <> '0') and (key <> '100') and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_40] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=500 width=269) - predicate:((key1 <> '0') and (key1 <> '100') and key2 is not null) - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + Select Operator [SEL_37] (rows=475 width=778) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_36] (rows=475 width=778) + predicate:(key2 is not null and (key1 <> '0') and (key1 <> '100')) + TableScan [TS_0] (rows=500 width=778) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_40] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_39] (rows=500 width=178) + predicate:((key <> '0') and (key <> '100') and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -237,40 +237,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (CUSTOM_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_34] (rows=791 width=447) + Select Operator [SEL_34] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=791 width=447) + Top N Key Operator [TNK_32] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=791 width=447) - BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [CUSTOM_EDGE] vectorized, llap - MULTICAST [RS_28] + Map Join Operator [MAPJOIN_31] (rows=550 width=178) + Conds:RS_28._col0=SEL_30._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) - predicate:key1 is not null - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + Select Operator [SEL_27] (rows=475 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_26] (rows=475 width=525) + predicate:key1 is not null + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_30] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_29] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -344,15 +344,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=778) Number of rows:20 - Select Operator [SEL_34] (rows=791 width=447) + Select Operator [SEL_34] (rows=550 width=778) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=791 width=447) + Top N Key Operator [TNK_32] (rows=550 width=778) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=791 width=447) + Map Join Operator [MAPJOIN_31] (rows=550 width=778) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] @@ -363,12 +363,12 @@ Stage-0 predicate:((key <> '0') and (key <> '100')) TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) + <-Select Operator [SEL_30] (rows=500 width=778) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) + Filter Operator [FIL_29] (rows=500 width=778) predicate:((key1 <> '0') and (key1 <> '100')) - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + TableScan [TS_0] (rows=500 width=778) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -435,40 +435,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (CUSTOM_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_34] (rows=791 width=447) + Select Operator [SEL_34] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=791 width=447) + Top N Key Operator [TNK_32] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=791 width=447) - BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [CUSTOM_EDGE] vectorized, llap - MULTICAST [RS_28] + Map Join Operator [MAPJOIN_31] (rows=550 width=178) + Conds:RS_28._col0=SEL_30._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) - predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null) - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + Select Operator [SEL_27] (rows=475 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_26] (rows=475 width=525) + predicate:(key1 is not null and (key2 <> 'val_0') and (key2 <> 'val_100')) + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_30] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_29] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -533,40 +533,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=447) + Limit [LIM_35] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_34] (rows=814 width=447) + Select Operator [SEL_34] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=814 width=447) + Top N Key Operator [TNK_32] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=814 width=447) - Conds:SEL_30._col1=RS_28._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [BROADCAST_EDGE] vectorized, llap + Map Join Operator [MAPJOIN_31] (rows=550 width=178) + Conds:RS_28._col1=SEL_30._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_28] PartitionCols:_col1 - Select Operator [SEL_27] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=269) - predicate:key2 is not null - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] + Select Operator [SEL_27] (rows=475 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_26] (rows=475 width=525) + predicate:key2 is not null + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_30] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_29] (rows=500 width=178) + predicate:value is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -629,40 +629,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (CUSTOM_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_51] - Limit [LIM_50] (rows=20 width=447) + Limit [LIM_50] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_49] (rows=791 width=447) + Select Operator [SEL_49] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_48] - Top N Key Operator [TNK_47] (rows=791 width=447) + Top N Key Operator [TNK_47] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_46] (rows=791 width=447) - BucketMapJoin:true,Conds:SEL_45._col0, _col1, _col2=RS_43._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [CUSTOM_EDGE] vectorized, llap - MULTICAST [RS_43] - PartitionCols:_col0, _col1 - Select Operator [SEL_42] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=500 width=178) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_45] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_44] (rows=500 width=269) - predicate:(key1 is not null and key2 is not null and value is not null) - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] + Map Join Operator [MAPJOIN_46] (rows=550 width=178) + Conds:RS_43._col0, _col1, _col2=SEL_45._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap + BROADCAST [RS_43] + PartitionCols:_col0, _col1, _col2 + Select Operator [SEL_42] (rows=425 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_41] (rows=425 width=525) + predicate:(key1 is not null and key2 is not null and value is not null) + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_45] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_44] (rows=500 width=178) + predicate:(key is not null and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -725,40 +725,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 2 vectorized, llap + Reducer 3 vectorized, llap File Output Operator [FS_41] - Limit [LIM_40] (rows=20 width=447) + Limit [LIM_40] (rows=20 width=178) Number of rows:20 - Select Operator [SEL_39] (rows=814 width=447) + Select Operator [SEL_39] (rows=550 width=178) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] vectorized, llap + <-Map 2 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_38] - Top N Key Operator [TNK_37] (rows=814 width=447) + Top N Key Operator [TNK_37] (rows=550 width=178) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_36] (rows=814 width=447) - Conds:SEL_35._col2=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 3 [BROADCAST_EDGE] vectorized, llap + Map Join Operator [MAPJOIN_36] (rows=550 width=178) + Conds:RS_33._col2=SEL_35._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_33] - PartitionCols:_col1 - Select Operator [SEL_32] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_31] (rows=500 width=178) + PartitionCols:_col2 + Select Operator [SEL_32] (rows=475 width=525) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_31] (rows=475 width=525) predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_35] (rows=500 width=269) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_34] (rows=500 width=269) + TableScan [TS_0] (rows=500 width=525) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] + <-Select Operator [SEL_35] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_34] (rows=500 width=178) predicate:value is not null - TableScan [TS_0] (rows=500 width=269) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: SELECT * FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out index 148f2d89ee20..2c4de1e43e85 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out @@ -148,11 +148,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=16 width=8) + Select Operator [SEL_31] (rows=17 width=8) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=16 width=8) + Map Join Operator [MAPJOIN_29] (rows=17 width=8) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -168,7 +168,7 @@ Stage-0 Filter Operator [FIL_27] (rows=16 width=8) predicate:key1 is not null TableScan [TS_0] (rows=16 width=8) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] PREHOOK: query: SELECT a.key1, a.id FROM srcbucket_big a @@ -232,11 +232,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=16 width=92) + Select Operator [SEL_31] (rows=17 width=194) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=16 width=92) + Map Join Operator [MAPJOIN_29] (rows=17 width=194) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -247,12 +247,12 @@ Stage-0 predicate:key2 is not null TableScan [TS_3] (rows=6 width=72) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key2"] - <-Select Operator [SEL_28] (rows=16 width=92) + <-Select Operator [SEL_28] (rows=16 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=92) + Filter Operator [FIL_27] (rows=16 width=188) predicate:key2 is not null - TableScan [TS_0] (rows=16 width=92) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] + TableScan [TS_0] (rows=16 width=188) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] PREHOOK: query: SELECT a.key2, a.id FROM srcbucket_big a @@ -316,11 +316,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_37] - Select Operator [SEL_36] (rows=16 width=77) + Select Operator [SEL_36] (rows=22 width=188) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_35] - Map Join Operator [MAPJOIN_34] (rows=16 width=77) + Map Join Operator [MAPJOIN_34] (rows=22 width=188) Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1"] <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_31] @@ -331,12 +331,12 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=6 width=74) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Select Operator [SEL_33] (rows=16 width=77) + <-Select Operator [SEL_33] (rows=20 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=16 width=77) + Filter Operator [FIL_32] (rows=20 width=188) predicate:value is not null - TableScan [TS_0] (rows=21 width=77) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["value","id"] + TableScan [TS_0] (rows=21 width=188) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["value","id"] PREHOOK: query: SELECT a.value, a.id FROM srcbucket_big a @@ -400,11 +400,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_42] - Select Operator [SEL_41] (rows=12 width=96) + Select Operator [SEL_41] (rows=13 width=194) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_40] - Map Join Operator [MAPJOIN_39] (rows=12 width=96) + Map Join Operator [MAPJOIN_39] (rows=13 width=194) BucketMapJoin:true,Conds:SEL_38._col0, _col1=RS_36._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_36] @@ -415,12 +415,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null) TableScan [TS_3] (rows=6 width=75) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2"] - <-Select Operator [SEL_38] (rows=12 width=96) + <-Select Operator [SEL_38] (rows=12 width=192) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=12 width=96) + Filter Operator [FIL_37] (rows=12 width=192) predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=96) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] + TableScan [TS_0] (rows=12 width=192) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] PREHOOK: query: SELECT a.key1, a.key2, a.id FROM srcbucket_big a @@ -480,11 +480,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_47] - Select Operator [SEL_46] (rows=9 width=78) + Select Operator [SEL_46] (rows=17 width=198) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_45] - Map Join Operator [MAPJOIN_44] (rows=9 width=78) + Map Join Operator [MAPJOIN_44] (rows=17 width=198) BucketMapJoin:true,Conds:SEL_43._col0, _col1=RS_41._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_41] @@ -495,12 +495,12 @@ Stage-0 predicate:(key1 is not null and value is not null) TableScan [TS_3] (rows=6 width=77) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","value"] - <-Select Operator [SEL_43] (rows=12 width=83) + <-Select Operator [SEL_43] (rows=16 width=192) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=12 width=83) - predicate:(value is not null and key1 is not null) - TableScan [TS_0] (rows=16 width=81) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] + Filter Operator [FIL_42] (rows=16 width=192) + predicate:(key1 is not null and value is not null) + TableScan [TS_0] (rows=16 width=192) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] PREHOOK: query: SELECT a.key1, a.value, a.id FROM srcbucket_big a @@ -560,11 +560,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_52] - Select Operator [SEL_51] (rows=5 width=150) + Select Operator [SEL_51] (rows=13 width=381) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_50] - Map Join Operator [MAPJOIN_49] (rows=5 width=150) + Map Join Operator [MAPJOIN_49] (rows=13 width=381) BucketMapJoin:true,Conds:SEL_48._col0, _col1, _col2=RS_46._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_46] @@ -575,12 +575,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null and value is not null) TableScan [TS_3] (rows=6 width=150) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] - <-Select Operator [SEL_48] (rows=8 width=164) + <-Select Operator [SEL_48] (rows=12 width=376) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=8 width=164) - predicate:(value is not null and key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=164) - default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] + Filter Operator [FIL_47] (rows=12 width=376) + predicate:(key1 is not null and key2 is not null and value is not null) + TableScan [TS_0] (rows=12 width=376) + default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] PREHOOK: query: SELECT a.key1, a.key2, a.value, a.id FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out index b4b2ff455e58..30d0b3d9ee3a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out @@ -704,7 +704,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -737,18 +737,15 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ts), max(ts), count(1), count(ts), compute_bit_vector_hll(ts), min(dd), max(dd), count(dd), compute_bit_vector_hll(dd), min(ll), max(ll), count(ll), compute_bit_vector_hll(ll) - keys: iceberg_bucket(ll, 4) (type: int) minReductionHashAggr: 0.4 mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type: int), _col12 (type: bigint), _col13 (type: binary) + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -771,17 +768,16 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) - keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'TIMESTAMP' (type: string), _col1 (type: timestamp), _col2 (type: timestamp), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'DOUBLE' (type: string), _col6 (type: double), _col7 (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), 'LONG' (type: string), UDFToLong(_col10) (type: bigint), UDFToLong(_col11) (type: bigint), (_col3 - _col12) (type: bigint), COALESCE(ndv_compute_bit_vector(_col13),0) (type: bigint), _col13 (type: binary), named_struct('ll_bucket',_col0) (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 907 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'TIMESTAMP' (type: string), _col0 (type: timestamp), _col1 (type: timestamp), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DOUBLE' (type: string), _col5 (type: double), _col6 (type: double), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 1 Data size: 863 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 907 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 863 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8e77ed6fd0a9..8c602ddb1975 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -383,14 +383,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v1 Snapshot ref: tag_v1 - Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -598,17 +598,17 @@ STAGE PLANS: alias: ice_orc.tag_v1 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v1 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -965,14 +965,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1008,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index 60ad0af240d3..3e8bda76a610 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -154,7 +154,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -239,7 +239,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -337,7 +337,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index b59449ea645f..f97e4c10212a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -148,7 +148,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -296,7 +296,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index add0220e2470..2fb957b5319f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -123,7 +123,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 3bb172c0df60..d140bbc30aa7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -82,7 +82,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 1 @@ -198,7 +198,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 466c234e72c5..0de0774396b6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -403,7 +403,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 4 @@ -490,7 +490,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out index 79536df6215d..a1eace49222e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out @@ -266,7 +266,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out index cefa4effc895..c41f2313aacb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out @@ -359,16 +359,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: o - filterExpr: ((((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70)) and itemid is not null) (type: boolean) - probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_29_container, bigKeyColName:itemid, smallTablePos:1, keyRatio:0.9523809523809523 - Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + filterExpr: (itemid is not null and (((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70))) (type: boolean) + probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_29_container, bigKeyColName:itemid, smallTablePos:1, keyRatio:1.0476190476190477 + Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70)) and itemid is not null) (type: boolean) - Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (itemid is not null and (((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70))) (type: boolean) + Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: quantity (type: int), itemid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -378,17 +378,17 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 3 - Statistics: Num rows: 21 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) - minReductionHashAggr: 0.95238096 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) @@ -420,10 +420,10 @@ STAGE PLANS: aggregations: sum(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index 9ed2e47d7837..9a99f74574ff 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -132,68 +132,68 @@ STAGE PLANS: TableScan alias: target_ice filterExpr: (a <= 100) (type: boolean) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a <= 100) (type: boolean) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Execution mode: vectorized Map 12 Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col3 (type: int) - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: FILE__PATH (type: string), a (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 10 @@ -202,7 +202,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -221,20 +221,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: @@ -245,17 +245,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 604 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 22 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 4 Execution mode: vectorized @@ -266,7 +266,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -281,20 +281,20 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col4 <> _col7) or _col4 is null or _col7 is null) (type: boolean) - Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL + predicate: (_col4 is null or (_col4 <> _col7) or _col7 is null) (type: boolean) + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: @@ -305,13 +305,13 @@ STAGE PLANS: 0 _col2 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 28 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Reduce Operator Tree: @@ -322,20 +322,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col1 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -355,26 +355,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -384,13 +384,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col3 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index c2529066adac..21800f645f62 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -96,20 +96,20 @@ STAGE PLANS: TableScan alias: target_ice filterExpr: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 @@ -121,21 +121,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -143,14 +143,14 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -158,52 +158,52 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: _col9 is null (type: boolean) - Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: (_col9 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) outputColumnNames: _col2, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz sort order: ++++ Map-reduce partition columns: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE value expressions: _col4 (type: bigint) Reducer 3 Execution mode: vectorized @@ -214,7 +214,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -229,7 +229,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -243,17 +243,17 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out index 9f1607322789..8b7776d7bae8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out @@ -404,10 +404,6 @@ hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### PREHOOK: query: select file from default.ice_meta_3.metadata_log_entries PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_3 @@ -708,10 +704,6 @@ hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### -hdfs://### HDFS PATH ### PREHOOK: query: select file from default.ice_meta_3.metadata_log_entries PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 0ff1c86f2107..6b157cd74b40 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -100,7 +100,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"orderid\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"quantity\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"itemid\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"tradets\",\"required\":false,\"type\":\"timestamp\"},{\"id\":5,\"name\":\"p1\",\"required\":false,\"type\":\"string\"},{\"id\":6,\"name\":\"p2\",\"required\":false,\"type\":\"string\"}]} @@ -266,7 +266,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Stage-3 Stats Work{} @@ -282,7 +282,7 @@ Stage-3 Select Operator [SEL_25] Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col4","_col5"] <-Reducer 2 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_21] + PARTITION_ONLY_SHUFFLE [RS_21] PartitionCols:_col4, _col5 Select Operator [SEL_20] (rows=11 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -298,15 +298,14 @@ Stage-3 default@llap_orders,llap_orders,Tbl:COMPLETE,Col:NONE,Output:["orderid","quantity","itemid","tradets","p1","p2"] Reducer 4 vectorized File Output Operator [FS_29] - Select Operator [SEL_28] (rows=5 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36"] - Group By Operator [GBY_27] (rows=5 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)","min(VALUE._col13)","max(VALUE._col14)","count(VALUE._col15)","compute_bit_vector_hll(VALUE._col16)","max(VALUE._col17)","avg(VALUE._col18)","count(VALUE._col19)","compute_bit_vector_hll(VALUE._col20)","max(VALUE._col21)","avg(VALUE._col22)","count(VALUE._col23)","compute_bit_vector_hll(VALUE._col24)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_24] - PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=11 width=420) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"],keys:p1, p2 + Select Operator [SEL_28] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] + Group By Operator [GBY_27] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)","min(VALUE._col13)","max(VALUE._col14)","count(VALUE._col15)","compute_bit_vector_hll(VALUE._col16)","max(VALUE._col17)","avg(VALUE._col18)","count(VALUE._col19)","compute_bit_vector_hll(VALUE._col20)","max(VALUE._col21)","avg(VALUE._col22)","count(VALUE._col23)","compute_bit_vector_hll(VALUE._col24)"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_24] + Group By Operator [GBY_23] (rows=1 width=1###) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"] Select Operator [SEL_22] (rows=11 width=420) Output:["orderid","quantity","itemid","tradets","p1","p2"] Please refer to the previous Select Operator [SEL_20] diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 6f3cfd8cadb7..522c86d5bf95 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -51,20 +51,20 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 909 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Map 4 @@ -93,26 +93,26 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reducer 3 Execution mode: vectorized @@ -123,7 +123,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -138,17 +138,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized @@ -156,7 +156,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -175,20 +175,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized @@ -196,7 +196,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col5 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -216,26 +216,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col5 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Union 2 Vertex: Union 2 @@ -392,86 +392,86 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(c) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Reducer 10 @@ -483,11 +483,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -498,13 +498,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -515,20 +515,20 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 13 Reduce Operator Tree: @@ -539,11 +539,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 14 Reduce Operator Tree: @@ -554,17 +554,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -575,11 +575,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 16 Reduce Operator Tree: @@ -590,24 +590,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 17 Reduce Operator Tree: @@ -618,11 +618,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 18 Reduce Operator Tree: @@ -633,13 +633,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 19 Reduce Operator Tree: @@ -650,16 +650,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 2 Execution mode: vectorized @@ -707,7 +707,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -726,20 +726,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 21 Execution mode: vectorized @@ -773,13 +773,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 23 Reduce Operator Tree: @@ -790,13 +790,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 24 Reduce Operator Tree: @@ -807,13 +807,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 25 Execution mode: vectorized @@ -835,21 +835,21 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 28 Execution mode: vectorized @@ -858,38 +858,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 29 Execution mode: vectorized @@ -898,11 +898,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -913,11 +913,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -928,13 +928,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 2110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -945,23 +945,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -981,26 +981,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 2321 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 1031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1010,17 +1010,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Execution mode: vectorized @@ -1031,7 +1031,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out index 01c7c92df472..9ea7240bd41b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out @@ -327,29 +327,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) @@ -371,12 +371,12 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -387,21 +387,21 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) - minReductionHashAggr: 0.7777778 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -412,7 +412,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.8888889 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -483,11 +483,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3496 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3496 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -498,24 +498,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3300 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3300 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -526,14 +526,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -565,11 +565,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 13 Data size: 2568 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1968 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 13 Data size: 2568 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1968 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -580,13 +580,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2048 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2048 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 16 Reduce Operator Tree: @@ -597,23 +597,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 13 Data size: 2672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 13 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -633,26 +633,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -679,11 +679,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -694,13 +694,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 20 Reduce Operator Tree: @@ -711,13 +711,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 21 Reduce Operator Tree: @@ -728,17 +728,17 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2528 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2781 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 927 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -769,38 +769,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 24 Execution mode: vectorized @@ -921,11 +921,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -936,13 +936,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3088 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3088 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -953,16 +953,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 3408 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 6 Execution mode: vectorized @@ -970,7 +970,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -989,17 +989,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1172 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1014,11 +1014,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 9 Reduce Operator Tree: @@ -1029,17 +1029,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Union 7 Vertex: Union 7 @@ -1155,69 +1155,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Map 14 @@ -1247,17 +1247,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -1268,20 +1268,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1301,26 +1301,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 13 Execution mode: vectorized Reduce Operator Tree: @@ -1328,11 +1328,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1343,11 +1343,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1358,24 +1358,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null) (type: boolean) + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1386,14 +1386,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1408,14 +1408,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 311 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1430,13 +1430,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized @@ -1444,7 +1444,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1463,17 +1463,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1488,47 +1488,47 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1661,69 +1661,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1735,20 +1735,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1768,26 +1768,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -1797,13 +1797,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 13 Execution mode: vectorized @@ -1811,7 +1811,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1830,17 +1830,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1855,60 +1855,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1918,14 +1918,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1938,17 +1938,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) Reducer 6 Reduce Operator Tree: @@ -1959,11 +1959,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -1974,24 +1974,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null) (type: boolean) + Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 8 Reduce Operator Tree: @@ -2002,14 +2002,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2022,11 +2022,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 4 Vertex: Union 4 @@ -2099,14 +2099,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), 'All' (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 855 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 855 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out index ccf73acb8669..a33dcfab92b3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out index 02111ac58daf..7ba771285ca4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out @@ -234,17 +234,17 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: (a = 22) (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 485 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -316,21 +316,21 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: (c = 66) (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (c = 66) (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col8 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -339,10 +339,10 @@ STAGE PLANS: Select Operator expressions: 33 (type: int), _col8 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -451,20 +451,20 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: a is not null (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 @@ -476,21 +476,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2465 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2465 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 397 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -498,14 +498,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -513,14 +513,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 482 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 482 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -528,14 +528,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: _col9 is null (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1479 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -543,24 +543,24 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: (_col9 = _col1) (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) outputColumnNames: _col2, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz sort order: ++++ Map-reduce partition columns: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint) Reducer 3 Execution mode: vectorized @@ -570,17 +570,17 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -795,14 +795,14 @@ STAGE PLANS: TableScan alias: ice01.branch_test1 Snapshot ref: branch_test1 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 0c4a56061cb7..da691dcaf79c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -280,11 +280,18 @@ private void addStatsForPartitionKeyColumn(Table table, List colStats, String colName, Map tableProps) throws HiveException { + if (table.isNonNative() && !StatsUtils.isPartitionStats(table, context.getConf())) { + // the table maintains table-level column statistics only, whose accuracy the table + // properties already reflect + colStats.addAll(context.getDb().getTableColumnStatistics(table, + Lists.newArrayList(colName.toLowerCase()), false)); + return; + } List parts = context.getDb().getPartitionNames(table, (short) -1); AggrStats aggrStats = context.getDb().getAggrColStatsFor(table, Lists.newArrayList(colName.toLowerCase()), parts, false); colStats.addAll(aggrStats.getColStats()); - + if (parts.size() == aggrStats.getPartsFound()) { StatsSetupConst.setColumnStatsState(tableProps, Lists.newArrayList(colName.toLowerCase())); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index bd9735100015..d7df1b039f98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -126,7 +126,10 @@ public int execute() { } private Table getTable(Hive db) throws SemanticException, HiveException { - return db.getTable(work.getFullTableName()); + Table table = db.getTable(work.getFullTableName()); + // a lookup by name carries no snapshot ref: restore the one the write targeted + table.setSnapshotRef(work.getTable().getSnapshotRef()); + return table; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index e4b3bb9eb836..ae0586093762 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -6310,6 +6310,12 @@ public List getTableColumnStatistics( if (tbl.isNonNative() && tbl.getStorageHandler().canProvideColStatistics(tbl)) { return tbl.getStorageHandler().getColStatistics(tbl, colNames); } + if (tbl.isNonNative() && (tbl.getStorageHandler().canSetColStatistics(tbl) + || !tbl.getQualifier().isEmpty())) { + // the handler owns the table's statistics, or the read is qualified by a branch, a + // point in time or a metadata table: the metastore's single set describes none of them + return Collections.emptyList(); + } if (checkTransactional) { AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl); retv = getMSC().getTableColumnStatistics(tbl.getDbName(), tbl.getTableName(), colNames, @@ -6364,6 +6370,12 @@ public AggrStats getAggrColStatsFor(Table tbl, if (tbl.isNonNative() && tbl.getStorageHandler().canProvideColStatistics(tbl)) { return tbl.getStorageHandler().getAggrColStatsFor(tbl, colNames, partName); } + if (tbl.isNonNative() && (tbl.getStorageHandler().canSetColStatistics(tbl) + || !tbl.getQualifier().isEmpty())) { + // the handler owns the table's statistics, or the read is qualified by a branch, a + // point in time or a metadata table: the metastore's single set describes none of them + return new AggrStats(new ArrayList<>(), 0); + } if (checkTransactional) { AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl); writeIdList = tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null; @@ -6377,7 +6389,7 @@ public AggrStats getAggrColStatsFor(Table tbl, perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.HIVE_GET_AGGR_COL_STATS, "HS2-cache"); } } - + public void deleteColumnStatistics(TableName tableName) throws HiveException { DeleteColumnStatisticsRequest request = new DeleteColumnStatisticsRequest(tableName.getDb(), tableName.getTable()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index ad18304fa72b..6ff3742833bc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -604,9 +604,6 @@ default List getPartitionTransformSpec(org.apache.hadoop.hive.ql. return null; } - default Map> getPartitionTransformSpecs(org.apache.hadoop.hive.ql.metadata.Table table) { - return null; - } /** * Creates a DynamicPartitionCtx instance that will be set up by the storage handler itself. Useful for non-native diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index a46e8c2cb95a..3cdfba01c97a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -285,7 +285,9 @@ public void checkValidity(Configuration conf) throws HiveException { // check for validity validateName(conf); - if (getCols().isEmpty()) { + if (getCols().isEmpty() && !hasNonNativePartitionSupport()) { + // a non-native table's data-column view excludes the handler partition columns and is + // legitimately empty when every column is a partition transform source throw new HiveException("at least one column must be specified for the table"); } validateColumns(getCols(), getPartCols(), DDLUtils.isIcebergTable(this)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index fe477303cadb..fa5317271678 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1537,6 +1537,11 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, } } + // the lookups above carry no snapshot ref: take it from the table the file sink writes to + if (table != null && nd.getConf().getTable() != null) { + table.setSnapshotRef(nd.getConf().getTable().getSnapshotRef()); + } + StatsWork columnStatsWork = new StatsWork(table, statsWork, hconf); columnStatsWork.collectStatsFromAggregator(nd.getConf()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index fee103ec4f09..5adff4b08655 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -68,7 +68,6 @@ public class ColumnStatsAutoGatherContext { private final List partitionColumns; private boolean isInsertInto; private Table tbl; - private List partTransformSpec; private Map partSpec; private Context origCtx; @@ -124,18 +123,14 @@ public void insertAnalyzePipeline() throws SemanticException { */ public void insertTableValuesAnalyzePipeline() throws SemanticException { // Instead of starting from analyze statement, we just generate the Select plan - boolean isPartitionStats = StatsUtils.isPartitionStats(tbl, conf); + boolean isPartitionStats = StatsUtils.isPartitionStats(tbl, conf) && !tbl.hasNonNativePartitionSupport(); if (isPartitionStats) { partSpec = new HashMap<>(); List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); partKeys.forEach(k -> partSpec.put(k, null)); - if (tbl.hasNonNativePartitionSupport()) { - partTransformSpec = tbl.getStorageHandler().getPartitionTransformSpec(tbl); - } } - String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery( - tbl, conf, partTransformSpec, partSpec, isPartitionStats); + String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, conf, partSpec, isPartitionStats); insertAnalyzePipeline(command, true); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index b82b21068a6c..33f19de294c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -20,18 +20,16 @@ package org.apache.hadoop.hive.ql.parse; import static org.apache.hadoop.hive.ql.metadata.HiveUtils.unparseIdentifier; -import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_SPEC_ID; +import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_NAME; import com.google.common.base.Preconditions; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import com.google.common.collect.Maps; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -176,8 +174,7 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } } - private static CharSequence genPartitionClause(Table tbl, List partTransformSpec, int specId, - Map partSpec, HiveConf conf) { + private static CharSequence genPartitionClause(Table tbl, Map partSpec, HiveConf conf) { boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); StringBuilder whereClause = new StringBuilder(" where ").append( @@ -188,26 +185,15 @@ private static CharSequence genPartitionClause(Table tbl, List pa .collect(Collectors.joining(" and ")) ); - if (specId >= 0) { - whereClause.append((predPresent) ? " and " : "") - .append(unparseIdentifier(PARTITION_SPEC_ID.getName(), conf) + "=" + specId); - predPresent = true; - } - - StringBuilder groupByClause = new StringBuilder(" group by ").append(( - (partTransformSpec != null) ? - partTransformSpec.stream().map(spec -> spec.toHiveExpr(conf)) : - tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) - ) - .collect(Collectors.joining(", ")) + StringBuilder groupByClause = new StringBuilder(" group by ").append( + tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) + .collect(Collectors.joining(", ")) ); // attach the predicate and group by to the return clause return predPresent ? whereClause.append(groupByClause) : groupByClause; } - - private static String getColTypeOf(Table tbl, String partKey) { for (FieldSchema fs : tbl.getPartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { @@ -261,10 +247,8 @@ protected static List getFieldSchemasByColName(Table tbl, List partTransformSpec, int specId, Map partSpec, - boolean isPartitionStats) { - String rewritten = genRewrittenQuery(tbl, columnSchemas, conf, partTransformSpec, specId, partSpec, - isPartitionStats, false); + Map partSpec, boolean isPartitionStats) { + String rewritten = genRewrittenQuery(tbl, columnSchemas, conf, partSpec, isPartitionStats, false); isRewritten = true; return rewritten; } @@ -274,15 +258,13 @@ private String genRewrittenQuery(FieldSchemas columnSchemas, HiveConf conf, * included in the input table. */ protected static String genRewrittenQuery(Table tbl, - HiveConf conf, List partTransformSpec, Map partSpec, - boolean isPartitionStats) { + HiveConf conf, Map partSpec, boolean isPartitionStats) { return ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, getStatsEligibleFieldSchemas(tbl), conf, - partTransformSpec, -1, partSpec, isPartitionStats, true); + partSpec, isPartitionStats, true); } private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, - HiveConf conf, List partTransformSpec, int specId, Map partSpec, - boolean isPartitionStats, boolean useTableValues) { + HiveConf conf, Map partSpec, boolean isPartitionStats, boolean useTableValues) { StringBuilder rewrittenQueryBuilder = new StringBuilder("select "); StringBuilder columnNamesBuilder = new StringBuilder(); @@ -311,7 +293,10 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, } if (isPartitionStats) { - if (partTransformSpec == null) { + if (tbl.hasNonNativePartitionSupport()) { + // group every row by its read-side partition name + rewrittenQueryBuilder.append(", ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + } else { for (FieldSchema fs : tbl.getPartCols()) { String identifier = unparseIdentifier(fs.getName(), conf); rewrittenQueryBuilder.append(", ").append(identifier); @@ -321,9 +306,6 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, .append(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()).toString()) .append(")"); } - } else { - rewrittenQueryBuilder.append(", ") - .append(TransformSpec.toNamedStruct(partTransformSpec, conf)); } } @@ -341,19 +323,17 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, .append(")"); } else { rewrittenQueryBuilder.append(unparseIdentifier(tbl.getDbName(), conf)) - .append(".") - .append(unparseIdentifier(tbl.getTableName(), conf)); - - if (tbl.getMetaTable() != null) { - rewrittenQueryBuilder.append(".") - .append(unparseIdentifier(tbl.getMetaTable(), conf)); - } + .append(".").append(unparseIdentifier(tbl.getTableName(), conf)); } // If partition level statistics is requested, add predicate and group by as needed to rewritten // query if (isPartitionStats) { - rewrittenQueryBuilder.append(genPartitionClause(tbl, partTransformSpec, specId, partSpec, conf)); + if (!tbl.hasNonNativePartitionSupport()) { + rewrittenQueryBuilder.append(genPartitionClause(tbl, partSpec, conf)); + } else { + rewrittenQueryBuilder.append("\ngroup by ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + } } String rewrittenQuery = rewrittenQueryBuilder.toString(); @@ -636,7 +616,6 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { // partition spec is rejected; the auto-gather path merges instead and stays unaffected) validateUnsupportedPartitionClause(tbl, AnalyzeCommandUtils.isPartitionLevelStats(ast)); - Map> partTransformSpecs = Collections.singletonMap(-1, null); Map partSpec = (isPartitionStats) ? AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf) : null; @@ -644,18 +623,10 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { if (isPartitionStats) { handlePartialPartitionSpec(partSpec, null); - if (tbl.hasNonNativePartitionSupport()) { - partTransformSpecs = tbl.getStorageHandler().getPartitionTransformSpecs(tbl); - } } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - - rewrittenQuery = String.join(" union all ", - Maps.transformEntries(partTransformSpecs, (specId, partTransformSpec) -> - genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, specId, partSpec, isPartitionStats)) - .values()); - + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); } else { // Not an analyze table column compute statistics statement - don't do any rewrites @@ -703,10 +674,9 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) tbl = AnalyzeCommandUtils.getTable(ast, this); - boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) - || StatsUtils.isPartitionStats(tbl, conf); - - List partTransformSpec = null; + boolean isPartitionStats = (AnalyzeCommandUtils.isPartitionLevelStats(ast) + || StatsUtils.isPartitionStats(tbl, conf)) && !tbl.hasNonNativePartitionSupport(); + Map partSpec = null; List columnSchemas = getColumnsFromAst(ast); @@ -714,15 +684,11 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) if (isPartitionStats) { partSpec = AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf); handlePartialPartitionSpec(partSpec, context); - if (tbl.hasNonNativePartitionSupport()) { - partTransformSpec = tbl.getStorageHandler().getPartitionTransformSpec(tbl); - } } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, -1, - partSpec, isPartitionStats); + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); return rewrittenTree; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java index d8f6898fef9e..bea31ba62f9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java @@ -41,8 +41,6 @@ public enum TransformType { private TransformType transformType; private Integer transformParam; - private String fieldName; - public TransformSpec() { } @@ -76,13 +74,6 @@ public void setTransformParam(Integer transformParam) { this.transformParam = transformParam; } - public void setFieldName(String fieldName) { - this.fieldName = fieldName; - } - - public String getFieldName() { - return fieldName; - } public String transformTypeString() { if (transformType == null) { @@ -92,13 +83,6 @@ public String transformTypeString() { "[" + width + "]").orElse(""); } - public static String toNamedStruct(List partTransformSpec, Configuration conf) { - return "named_struct(" + - partTransformSpec.stream().map(spec -> - "'" + spec.getFieldName() + "', " + spec.toHiveExpr(conf)) - .collect(Collectors.joining(", ")) + - ")"; - } public String toHiveExpr(Configuration conf) { String identifier = unparseIdentifier(columnName, conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 55072575a08e..239105fad78d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -315,7 +315,7 @@ private int aggregateStats(Hive db, Table tbl) { } } - List partitions = getPartitionsList(db); + List partitions = getPartitionsList(db, tbl); String tableFullName = table.getDbName() + "." + table.getTableName(); @@ -509,17 +509,22 @@ private String toString(Map parameters) { * @return a list of partitions that need to update statistics. * @throws HiveException */ - private List getPartitionsList(Hive db) throws HiveException { + private List getPartitionsList(Hive db, Table tbl) throws HiveException { if (work.getLoadFileDesc() != null) { return null; //we are in CTAS, so we know there are no partitions } + // the handler keeps this table's partitions: return before the branches replace the task's table + if (tbl.hasNonNativePartitionSupport()) { + return null; + } + if (work.getTableSpecs() != null) { // ANALYZE command TableSpec tblSpec = work.getTableSpecs(); table = tblSpec.tableHandle; - if (!table.isPartitioned() || table.hasNonNativePartitionSupport()) { + if (!table.isPartitioned()) { return null; } // get all partitions that match with the partition spec @@ -529,7 +534,7 @@ private List getPartitionsList(Hive db) throws HiveException { // INSERT OVERWRITE command LoadTableDesc tbd = work.getLoadTableDesc(); table = db.getTable(tbd.getTable().getTableName()); - if (!table.isPartitioned() || table.hasNonNativePartitionSupport()) { + if (!table.isPartitioned()) { return null; } DynamicPartitionCtx dpCtx = tbd.getDPCtx(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 45c5dc6cdd6d..15d23a8e329b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -64,7 +64,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class ColStatsProcessor implements IStatsProcessor { private static final Logger LOG = LoggerFactory.getLogger(ColStatsProcessor.class); @@ -142,22 +141,15 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List partColSchema = new ArrayList<>(); - List partVals = new ArrayList<>(); - if (tbl.hasNonNativePartitionSupport()) { - ObjectInspector inspector = fields.get(pos).getFieldObjectInspector(); - if (inspector.getCategory() == ObjectInspector.Category.STRUCT) { - Object obj = values.get(pos); - StructObjectInspector oi = (StructObjectInspector) inspector; - - for (StructField field : oi.getAllStructFieldRefs()) { - partColSchema.add(new FieldSchema(field.getFieldName(), null, "")); - partVals.add(String.valueOf(oi.getStructFieldData(obj, field))); - } + // an ANALYZE rewrite groups by the read-side partition name the reader materialized + if (fields.get(pos).getFieldObjectInspector() instanceof PrimitiveObjectInspector poi) { + Object partVal = poi.getPrimitiveJavaObject(values.get(pos)); + partName = partVal == null ? null : partVal.toString(); } } else { - partColSchema.addAll(tbl.getPartCols()); + List partColSchema = new ArrayList<>(tbl.getPartCols()); + List partVals = new ArrayList<>(); // Iterate over partition columns to figure out partition name for (int i = pos; i < pos + partColSchema.size(); i++) { Object partVal = ((PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector()) @@ -165,8 +157,8 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List colStats = new ArrayList<>(); @@ -237,7 +236,7 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce } start = System. currentTimeMillis(); - if (tbl.isNonNative() && tbl.getStorageHandler().canSetColStatistics(tbl)) { + if (useStorageHandler) { boolean success = tbl.getStorageHandler().setColStatistics(tbl, colStats); if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary())) { setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); @@ -256,7 +255,10 @@ public void setDpPartSpecs(Collection dpPartSpecs) { } private void setOrRemoveColumnStatsAccurateProperty(Hive db, Table tbl, List colNames, boolean success) throws HiveException { - if (CollectionUtils.isEmpty(colNames) || !colStatDesc.isTblLevel()) { + // a storage handler table has no HMS partition objects to carry per-partition flags: + // the table-level flag summarizes its partition statistics as a whole + boolean tableLevelFlag = colStatDesc.isTblLevel() || tbl.isNonNative(); + if (CollectionUtils.isEmpty(colNames) || !tableLevelFlag) { return; } EnvironmentContext environmentContext = new EnvironmentContext(); @@ -370,7 +372,6 @@ public enum ColumnStatsType { ColumnStatsField.BITVECTOR, ColumnStatsField.KLL_SKETCH)); - private final List columnStats; ColumnStatsType(List columnStats) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 1fbe66a4ecf4..d3e489f4a7c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -392,6 +392,34 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p } if (needColStats) { + + if (table.isNonNative() && !isPartitionStats(table, conf)) { + // the table maintains table-level column statistics only: serve them over the pruned + // set, on top of the partition-derived basic statistics + List colStats = getTableColumnStats(table, neededColumns, colStatsCache, fetchColStats); + if (estimateStats) { + colStats = estimateStatsForMissingCols(neededColumns, colStats, conf, nr, schema); + } + // we should have stats for all columns (estimated or actual) + if (neededColumns.size() == colStats.size()) { + long betterDS = getDataSizeFromColumnStats(nr, colStats); + if (betterDS >= 1 && !colStats.isEmpty()) { + stats.setDataSize(betterDS); + } + } + // infer if any column can be primary key based on column statistics + inferAndSetPrimaryKey(stats.getNumRows(), colStats); + + stats.setColumnStatsState(deriveStatType(colStats, neededColumns)); + stats.addToColumnStats(colStats); + + if (partStats.isEmpty()) { + // all partitions are filtered by partition pruning + stats.setBasicStatsState(State.COMPLETE); + } + return stats; + } + List partitionCols = getPartitionColumns(schema, neededColumns, referencedColumns); // We will retrieve stats from the metastore only for columns that are not cached @@ -441,7 +469,9 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p addPartitionColumnStats(conf, partitionColsToRetrieve, schema, partList, columnStats); long betterDS = getDataSizeFromColumnStats(nr, columnStats); - stats.setDataSize((betterDS < 1 || columnStats.isEmpty()) ? ds : betterDS); + if (betterDS >= 1 && !columnStats.isEmpty()) { + stats.setDataSize(betterDS); + } // infer if any column can be primary key based on column statistics inferAndSetPrimaryKey(stats.getNumRows(), columnStats); From 305b64e84a59693584e13bab0397427c37418af3 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 21 Aug 2026 20:48:59 +0300 Subject: [PATCH 04/15] HIVE-29829: Let a table choose its column-statistics granularity, and keep a branch's on the branch hive.iceberg.stats.collect.partlevel (default false) chooses the granularity for Iceberg tables, apart from the generic hive.analyze.stmt.collect.partlevel.stats. ANALYZE produces partition-level statistics; every other write maintains table-level ones. The read path keys on the calling API rather than on session config, so a granularity that changed between a write and a read cannot surface a blob of the wrong shape. A write to a branch stores its statistics on that branch's snapshot and leaves the table-scoped metastore row alone, including on the footer scan that ANALYZE ... COMPUTE STATISTICS uses. A whole-table INSERT OVERWRITE of a partitioned table now replaces its column statistics. It skipped before, because the snapshot summary does not distinguish a whole-table overwrite from a partition-scoped one. --- .../org/apache/hadoop/hive/conf/HiveConf.java | 7 + .../mr/hive/HiveIcebergStorageHandler.java | 18 +- .../iceberg/mr/hive/IcebergTableUtil.java | 78 +-- .../mr/hive/stats/ColStatsWritePolicy.java | 77 ++- .../mr/hive/TestHiveIcebergStatistics.java | 235 ++++++- .../positive/iceberg_colstats_granularity.q | 37 ++ .../results/positive/bucket_map_join_9.q.out | 18 +- .../src/test/results/positive/col_stats.q.out | 2 +- .../ctas_iceberg_partitioned_orc.q.out | 2 +- ...te_iceberg_copy_on_write_partitioned.q.out | 494 +++++++------- .../positive/dynamic_partition_pruning.q.out | 608 +++++++++--------- .../positive/dynamic_semijoin_reduction.q.out | 42 +- .../iceberg_colstats_granularity.q.out | 326 ++++++++++ .../iceberg_insert_into_partition.q.out | 342 +++++----- ...erg_insert_into_partition_transforms.q.out | 172 ++--- ...insert_into_partition_with_evolution.q.out | 2 +- .../iceberg_insert_overwrite_partition.q.out | 332 +++++----- ...nsert_overwrite_partition_transforms.q.out | 146 ++--- .../iceberg_partition_pruner_cache_key.q.out | 40 +- .../positive/iceberg_pcr_null_partition.q.out | 12 +- .../positive/iceberg_stats_with_ppr.q.out | 8 +- ...rg_truncate_partition_with_evolution.q.out | 52 +- .../llap/hadoop_catalog_create_table.q.out | 6 +- .../llap/iceberg_bucket_map_join_1.q.out | 32 +- .../llap/iceberg_bucket_map_join_2.q.out | 52 +- .../llap/iceberg_bucket_map_join_3.q.out | 36 +- .../llap/iceberg_bucket_map_join_4.q.out | 50 +- .../llap/iceberg_bucket_map_join_5.q.out | 36 +- .../llap/iceberg_bucket_map_join_6.q.out | 26 +- .../llap/iceberg_bucket_map_join_7.q.out | 334 +++++----- .../llap/iceberg_bucket_map_join_8.q.out | 72 +-- ...major_compaction_partition_evolution.q.out | 54 +- ...mpaction_partition_evolution_ordered.q.out | 4 +- ...ion_single_partition_with_evolution2.q.out | 2 +- .../iceberg_minor_compaction_bucket.q.out | 4 +- .../positive/llap/llap_iceberg_read_orc.q.out | 24 +- ...ge_iceberg_copy_on_write_partitioned.q.out | 96 +-- .../merge_iceberg_partitioned_orc.q.out | 56 +- ...ceberg_metadata_of_partitioned_table.q.out | 8 + .../src/test/results/positive/row_count.q.out | 16 +- ...te_iceberg_copy_on_write_partitioned.q.out | 376 +++++------ .../update_iceberg_partitioned_avro.q.out | 4 +- .../update_iceberg_partitioned_orc.q.out | 4 +- .../update_iceberg_partitioned_parquet.q.out | 4 +- .../vectorized_iceberg_merge_mixed.q.out | 189 ++---- .../hive/ql/parse/AnalyzeCommandUtils.java | 11 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 10 +- .../hive/ql/stats/BasicStatsNoJobTask.java | 7 +- .../hadoop/hive/ql/stats/BasicStatsTask.java | 5 +- .../hive/ql/stats/ColStatsProcessor.java | 5 +- .../hadoop/hive/ql/stats/StatsUtils.java | 5 +- 51 files changed, 2556 insertions(+), 2022 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6268efa4e50a..ec1835e63f2b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2130,6 +2130,13 @@ public static enum ConfVars { "Whether to use codec pool in ORC. Disable if there are bugs with codec reuse."), HIVE_ICEBERG_STATS_SOURCE("hive.iceberg.stats.source", "iceberg", "Use stats from iceberg table snapshot for query planning. This has two values metastore and iceberg"), + HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL("hive.iceberg.stats.collect.partlevel", false, + "Whether column statistics of a partitioned Iceberg table are kept per partition, letting\n" + + "the planner estimate from the partitions a query scans. ANALYZE writes the partitions it\n" + + "reads, CTAS and INSERT OVERWRITE write the partitions they replace, and a major\n" + + "compaction of one current-spec partition refreshes it. Plain INSERT maintains no\n" + + "partition-level statistics: its partitions read as stale until recomputed. After changing\n" + + "this, statistics of the other granularity are ignored until recomputed."), HIVE_ICEBERG_EXPIRE_SNAPSHOT_NUMTHREADS("hive.iceberg.expire.snapshot.numthreads", 4, "The number of threads to be used for deleting files during expire snapshot. If set to 0 or below it uses the" + " default DirectExecutorService"), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 429bdcd4a3aa..8f1857f7f91e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -776,7 +776,7 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta filter = null; } - return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), conf, filter); + return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); } @Override @@ -797,7 +797,7 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms Predicate filter = metadata -> partitions.contains(metadata.properties().get(PARTITION)); - List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), conf, filter); + List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), true, filter); partStats.forEach(colStats -> colStats.getStatsObj().removeIf(statsObj -> !colNames.contains(statsObj.getColName()))); @@ -870,19 +870,13 @@ private String getStatsSource() { /** - * Extends a table-level increment with the previous snapshot's statistics - or lets it stand - * alone when no rows precede it (first write, truncate). Returns whether the write may proceed: - * without previous statistics to extend, the increment is dropped - only a complete rewrite - * can restart the chain. + * Extends a table-level increment with the previous snapshot's statistics. Returns whether the + * write may proceed: without previous statistics to extend, the increment is dropped - only a + * complete rewrite can restart the chain. */ private boolean checkAndMergeColStats(List statsNew, Table tbl, Snapshot snapshot) throws InvalidObjectException, IOException { - Long prevSnapshotId = snapshot.parentId(); - if (prevSnapshotId == null || IcebergTableUtil.isEmptySnapshot(tbl.snapshot(prevSnapshotId))) { - // an empty base: the increment is the complete statistics - return true; - } - StatisticsFile statsOldSrc = IcebergTableUtil.getColStatsFile(tbl, prevSnapshotId, conf); + StatisticsFile statsOldSrc = IcebergTableUtil.getColStatsFile(tbl, snapshot.parentId(), conf); if (statsOldSrc == null) { return false; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 1fe44a0aee5f..2c7ef7fe1938 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -271,48 +271,45 @@ static Snapshot getTableSnapshot(Table table, String snapshotRef) { return table.currentSnapshot(); } - /** The current snapshot's own column statistics file of the flag-maintained granularity. */ - static StatisticsFile getColStatsFile(Table table, Configuration conf) { - return lookupColStatsFile(table, table.currentSnapshot().snapshotId(), conf); - } - /** * The newest column statistics file describing the snapshot: its own, or - across commits that - * rewrite files without changing any rows (compaction) - an ancestor's. Only a file of the - * granularity the flag maintains is served: partition-level blobs carry the partition name in - * their metadata, table-level blobs don't. + * rewrite files without changing any rows (compaction) - an ancestor's. Of the granularity the + * table maintains, which is the one every write produces. */ - public static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { - for (Snapshot snapshot = table.snapshot(snapshotId); snapshot != null; - snapshot = snapshot.parentId() != null ? table.snapshot(snapshot.parentId()) : null) { - StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), conf); - if (statsFile != null) { - return statsFile; - } - if (!DataOperations.REPLACE.equals(snapshot.operation())) { - return null; - } - } - return null; + static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { + return colStatsFileOf( + table, snapshotId, isPartitionStats(table, conf), + snapshot -> !DataOperations.REPLACE.equals(snapshot.operation())); } /** * The nearest column statistics file describing the snapshot: its own, or the closest - * ancestor's. The lookup stops at a snapshot holding no rows (truncate): earlier statistics - * describe data that didn't survive it. Statistics of an ancestor describe an earlier state of - * the data, which {@link #colStatsAccurate} reports and the planner treats as partial. + * ancestor's. Statistics of an ancestor describe an earlier state of the data, which + * {@link #colStatsAccurate} reports and the planner treats as partial. */ static StatisticsFile findColStatsFile(Table table, long snapshotId, Configuration conf) { + return findColStatsFile(table, snapshotId, isPartitionStats(table, conf)); + } + + private static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + // a snapshot holding no rows (truncate) ends the walk: what precedes it didn't survive + return colStatsFileOf( + table, snapshotId, partitionLevel, IcebergTableUtil::isEmptySnapshot); + } + + /** The statistics file of the snapshot or of an ancestor, up to the one {@code last} names. */ + private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boolean partitionLevel, + Predicate last) { if (table.statisticsFiles().isEmpty()) { return null; } for (Snapshot snapshot = table.snapshot(snapshotId); snapshot != null; snapshot = snapshot.parentId() != null ? table.snapshot(snapshot.parentId()) : null) { - StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), conf); + StatisticsFile statsFile = lookupColStatsFile(table, snapshot.snapshotId(), partitionLevel); if (statsFile != null) { return statsFile; } - if (isEmptySnapshot(snapshot)) { + if (last.test(snapshot)) { return null; } } @@ -320,14 +317,15 @@ static StatisticsFile findColStatsFile(Table table, long snapshotId, Configurati } /** - * Only a file of the granularity the flag maintains is served: partition-level blobs carry - * the partition name in their metadata, table-level blobs don't. + * The file whose blobs are Hive's own - Iceberg keeps statistics of its own in the same format - + * at the asked-for granularity: a blob describing one partition names it in its metadata. */ - private static StatisticsFile lookupColStatsFile(Table table, long snapshotId, Configuration conf) { + private static StatisticsFile lookupColStatsFile(Table table, long snapshotId, boolean partitionLevel) { return table.statisticsFiles().stream() .filter(stats -> stats.snapshotId() == snapshotId) - .filter(stats -> stats.blobMetadata().stream().anyMatch(metadata -> - isHiveColStatsBlob(table, metadata, conf))) + .filter(stats -> stats.blobMetadata().stream().anyMatch( + metadata -> ColumnStatisticsObj.class.getSimpleName().equals(metadata.type()) && + metadata.properties().containsKey(PARTITION_FIELD) == partitionLevel)) .findAny().orElse(null); } @@ -339,12 +337,7 @@ public static boolean isEmptySnapshot(Snapshot snapshot) { /** Whether the table's column statistics are maintained per partition, as the flag directs. */ public static boolean isPartitionStats(Table table, Configuration conf) { - return HiveConf.getBoolVar(conf, ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && table.spec().isPartitioned(); - } - - private static boolean isHiveColStatsBlob(Table table, org.apache.iceberg.BlobMetadata metadata, Configuration conf) { - return ColumnStatisticsObj.class.getSimpleName().equals(metadata.type()) && - isPartitionStats(table, conf) == metadata.properties().containsKey(PARTITION_FIELD); + return HiveConf.getBoolVar(conf, ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL) && table.spec().isPartitioned(); } static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotId) { @@ -992,9 +985,18 @@ public static boolean colStatsAccurate(Table table, Snapshot snapshot, Configura return getColStatsFile(table, snapshot.snapshotId(), conf) != null; } - static List readColStats(Table table, long snapshotId, Configuration conf, + /** The stored statistics describing the whole table. */ + static List readColStats(Table table, long snapshotId, Predicate filter) { + return readColStats(table, snapshotId, false, filter); + } + + /** + * The stored statistics of the granularity the caller serves. Asking for one the table doesn't + * keep yields nothing rather than blobs of the other shape. + */ + static List readColStats(Table table, long snapshotId, boolean partitionLevel, Predicate filter) { - StatisticsFile statsFile = findColStatsFile(table, snapshotId, conf); + StatisticsFile statsFile = findColStatsFile(table, snapshotId, partitionLevel); if (statsFile == null) { LOG.warn("Column stats file not found for snapshot: {}", snapshotId); return Lists.newArrayList(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java index 6a95f6b5c584..f318c46dce9b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java @@ -35,76 +35,79 @@ import org.apache.iceberg.mr.hive.IcebergTableUtil; /** - * What a write does to a table's column statistics. The statistics file is written whole, so a - * write may replace it only when the statistics it computed cover the table, and may merge into - * it only when they cover exactly the rows the write added. + * What a write does to a table's column statistics. Iceberg keeps them in one file that is always + * written whole, so a write replaces that file, merges into it, or leaves it alone. */ public enum ColStatsWritePolicy { - /** write the computed statistics as the statistics file */ + /** Write the computed statistics, discarding the stored ones. */ REPLACE, - /** write the computed statistics extended with the previous ones */ + /** Write the computed statistics merged with the stored ones. */ MERGE, - /** write nothing: the stored statistics still describe the data */ + /** Write nothing: leave the stored statistics as they are. */ SKIP; /** - * The policy for the write that computed these statistics: the operation that ran it, the - * granularity the table maintains and what the current snapshot already carries decide it. + * What to do with the statistics a write computed. * - * @param snapshot the snapshot the write moved: the table's current one, or a branch's head - * @param colStats the computed statistics, table-level or one entry per partition + * @param snapshot the snapshot the statistics describe: the table's current one, or a branch head + * @param colStats the computed statistics, one entry for the table or one per partition */ - @SuppressWarnings("checkstyle:CyclomaticComplexity") public static ColStatsWritePolicy resolve( Table tbl, Snapshot snapshot, List colStats, Configuration conf) { if (!colStats.getFirst().getStatsDesc().isIsTblLevel()) { - // partition-level statistics have a single writer: a table-wide ANALYZE + // Only ANALYZE gathers stats for every partition. return isAnalyzeOperation(conf) ? REPLACE : SKIP; } if (IcebergTableUtil.isPartitionStats(tbl, conf)) { - // the gather covered the rows, not the granularity the table maintains + // This table's readers want partition-level stats, which only ANALYZE produces. return SKIP; } if (SessionState.get().isCompaction()) { - // compaction preserves rows, so accurate statistics stay exact; a whole-table major one - // has read every row already, making a refresh of stale statistics free + // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. return isFullTableMajorCompaction(conf) && !IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf) ? REPLACE : SKIP; } - if (isAnalyzeOperation(conf)) { - // a full recompute, and the one statement reaching here that modifies no rows - return REPLACE; - } + // ANALYZE reads the whole table, so it replaces. + return isAnalyzeOperation(conf) ? REPLACE : resolveForWrite(tbl, snapshot, conf); + } + + /** What an INSERT, INSERT OVERWRITE or CTAS does to the stored statistics. */ + private static ColStatsWritePolicy resolveForWrite(Table tbl, Snapshot snapshot, Configuration conf) { if (IcebergTableUtil.isEmptySnapshot(snapshot)) { - // before the row count: an overwrite that selects nothing empties the table, writing none + // The table is now empty, so stats of the rows it held must go. Must precede the + // wroteNoRows check, which an emptying write also matches. return REPLACE; } - if (wroteNoRows(conf)) { + if (wroteNoRows(conf) || IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf)) { + // Nothing to record: the statement wrote no rows, or this snapshot already has stats. return SKIP; } - if (IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf)) { - // a committed write leaves its snapshot without statistics until this call writes them, so - // a snapshot already owning them saw no write here: the fallback when no query state exists - return SKIP; + if (holdsOnlyAddedRows(snapshot)) { + // Every row came from this write: a CTAS, a whole-table INSERT OVERWRITE, or the first + // INSERT after a TRUNCATE. + return REPLACE; } if (Boolean.parseBoolean(snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP))) { - // a partition overwrite replaces a slice: its increment neither extends the previous - // statistics nor recomputes them, while a whole-table one recomputes them + // Iceberg flags whole-table and partition overwrites alike, so the partition spec decides: + // only an unpartitioned table had every row replaced. return tbl.spec().isPartitioned() ? SKIP : REPLACE; } + // An INSERT: its stats cover the rows it added, the stored ones cover the rest. return MERGE; } - private static boolean isFullTableMajorCompaction(Configuration conf) { - return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)) && - conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; - } - /** - * Whether the statement wrote no rows: the column statistics are gathered by the write itself, so - * by the time they are persisted the query state carries the row count its file sink reported. + * Whether the table holds nothing but the rows this snapshot added. Iceberg carries the row + * total across commits, so it stays above the added count while older rows remain. False if + * either count is missing from the summary. */ + private static boolean holdsOnlyAddedRows(Snapshot snapshot) { + String added = snapshot.summary().get(SnapshotSummary.ADDED_RECORDS_PROP); + return added != null && added.equals(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP)); + } + + /** Whether the statement wrote no rows, as its file sink reported to the query state. */ private static boolean wroteNoRows(Configuration conf) { return SessionStateUtil.getQueryState(conf) .map(qs -> qs.getNumModifiedRows() == 0) @@ -116,4 +119,10 @@ private static boolean isAnalyzeOperation(Configuration conf) { .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) .orElse(false); } + + /** Whether the compaction read the whole table: a minor one sets a file size threshold. */ + private static boolean isFullTableMajorCompaction(Configuration conf) { + return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)) && + conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index c60f4b6b5098..581c12c2f705 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -99,6 +99,8 @@ public static Collection parameters() { @Before public void setStatsSource() { HiveConf.setVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE, statsSource); + // these tests describe the per partition statistics an Iceberg table keeps when asked to + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); } @Test @@ -224,6 +226,38 @@ public void testBranchWriteLeavesTableStatsUntouched() { Map basicStats = storageHandler().getBasicStatistics(hmsTable(identifier)); Assert.assertEquals("3", basicStats.get(StatsSetupConst.ROW_COUNT)); Assert.assertEquals(Long.valueOf(3L), storageHandler().getRowCount(hmsTable(identifier))); + + // and the metastore parameters, which are table-scoped, still count the table's rows + Assert.assertEquals("3", hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + + // asked for the branch, the handler counts the branch's rows + org.apache.hadoop.hive.ql.metadata.Table branchHmsTable = hmsTable(identifier); + branchHmsTable.setSnapshotRef("branch_b1"); + Assert.assertEquals("4", + storageHandler().getBasicStatistics(branchHmsTable).get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(4L), storageHandler().getRowCount(branchHmsTable)); + } + + @Test + public void testAnalyzeOnBranchLeavesTableBasicStatsUntouched() { + // a plain analyze takes the footer scan path, whose row count describes the branch it named + TableIdentifier identifier = TableIdentifier.of("default", "customers_analyze_branch_basic"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + Assert.assertEquals("3", hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS"); + + Assert.assertEquals("the branch's row count is not the table's", "3", + hmsTable(identifier).getParameters().get(StatsSetupConst.ROW_COUNT)); + Assert.assertEquals(Long.valueOf(3L), storageHandler().getRowCount(hmsTable(identifier))); } @Test @@ -243,9 +277,18 @@ public void testBranchWriteStoresItsColStatsOnTheBranch() { shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); Table icebergTable = testTables.loadTable(identifier); + long branchSnapshotId = icebergTable.snapshot("b1").snapshotId(); Assert.assertNotNull("the branch's head carries the statistics its write gathered", - IcebergTableUtil.getColStatsFile( - icebergTable, icebergTable.snapshot("b1").snapshotId(), shell.getHiveConf())); + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + + // the increment extends the fork point's statistics: the table's 0..2 plus the branch's 100 + List branchStats = + IcebergTableUtil.readColStats(icebergTable, branchSnapshotId, partitionLevel(icebergTable), null); + ColumnStatisticsObj branchId = branchStats.stream() + .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); + Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(100L, branchId.getStatsData().getLongStats().getHighValue()); + checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); } @@ -265,13 +308,53 @@ public void testAnalyzeOnBranchStoresStatsOnTheBranch() { shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (100, \'Bob\', \'Brown\')"); + + // a write that gathers nothing: its head carries no statistics, and 500 is a value neither the + // table nor the statistics the previous write stored have ever seen + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (500, \'Cy\', \'Green\')"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + + long branchSnapshotId = testTables.loadTable(identifier).snapshot("b1").snapshotId(); + Assert.assertNull("the branch's head starts without statistics", IcebergTableUtil.getColStatsFile( + testTables.loadTable(identifier), branchSnapshotId, shell.getHiveConf())); + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS FOR COLUMNS"); + // the branch ANALYZE leaves the table's own statistics alone checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 2); + + // and it describes the branch's rows, stored on the branch's head Table icebergTable = testTables.loadTable(identifier); Assert.assertNotNull("the branch's head carries the statistics the analyze computed", - IcebergTableUtil.getColStatsFile( - icebergTable, icebergTable.snapshot("b1").snapshotId(), shell.getHiveConf())); + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + List branchStats = + IcebergTableUtil.readColStats(icebergTable, branchSnapshotId, partitionLevel(icebergTable), null); + ColumnStatisticsObj branchId = branchStats.stream() + .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); + Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals("only the branch holds this row", 500L, + branchId.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testBranchWriteLeavesTableColStatsAccuracyAlone() { + // COLUMN_STATS_ACCURATE describes the table, so a branch write must not restore it + assumeParquetHiveCatalogIceberg(); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + TableIdentifier identifier = TableIdentifier.of("default", "customers_branch_flag"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1),(2)"); + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Assert.assertFalse("the delete stales the table's statistics", colStatsAccurate(identifier)); + + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (9)"); + Assert.assertFalse("the branch write describes the branch, not the table", + colStatsAccurate(identifier)); } @Test @@ -396,6 +479,135 @@ public void testStatsWithPartitionedCTAS() { checkColStatMaxLengthDistinctValue("target", "name", 5, 3); } + @Test + public void testTableLevelColStatsForPartitionedCtas() { + // the create gathered statistics covering every row it wrote: at table granularity those are + // the table's own, so they stand without an analyze to follow + assumeParquetHiveCatalogIceberg(); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE TABLE source (id bigint, name string) PARTITIONED BY (dept string) STORED AS ORC"); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, TableIdentifier.of("default", "source"), false)); + + shell.executeStatement(String.format( + "CREATE TABLE target PARTITIONED BY (dept, name) STORED BY ICEBERG %s AS SELECT * FROM source s", + testTables.propertiesForCreateTableSQL( + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())))); + + checkColStat("target", "id", true); + checkColStatMinMaxValue("target", "id", 0, 2); + } + + @Test + public void testTableLevelColStatsForPartitionedInsertOverwrite() { + // an overwrite of the whole table recomputes every row, so at table granularity its statistics + // are the table's; one scoped to a partition leaves the rows of the others behind + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "iow_source"); + TableIdentifier identifier = TableIdentifier.of("default", "iow_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint, p string) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + source + " VALUES (11, \'a\'), (19, \'b\')"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) PARTITIONED BY (p string) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'a\') VALUES (1), (2)"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'b\') VALUES (7), (9)"); + + shell.executeStatement( + "INSERT OVERWRITE TABLE " + identifier + " PARTITION (p) SELECT id, p FROM " + source); + Assert.assertTrue("the overwrite rewrote every row", colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 11, 19); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " PARTITION (p=\'a\') VALUES (100)"); + Assert.assertFalse("a partition overwrite describes a slice, not the table", + colStatsAccurate(identifier)); + } + + @Test + public void testColStatsForInsertOverwriteEmptyingTheTable() { + // an overwrite that selects nothing empties an unpartitioned table: the statistics describing + // the rows it held must not outlive them + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "iow_empty_source"); + TableIdentifier identifier = TableIdentifier.of("default", "iow_empty_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (2), (3)"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 3); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " SELECT id FROM " + source); + + Assert.assertEquals("the overwrite left no rows", Long.valueOf(0L), + storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the empty table's statistics are its own", colStatsAccurate(identifier)); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + } + + @Test + public void testColStatsForCtasSelectingNoRows() { + // the create wrote no rows: statistics describing none are the ones the table has + assumeParquetHiveCatalogIceberg(); + + TableIdentifier source = TableIdentifier.of("default", "ctas_empty_source"); + TableIdentifier identifier = TableIdentifier.of("default", "ctas_empty_target"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + + shell.executeStatement("CREATE EXTERNAL TABLE " + source + + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + source + " VALUES (1), (2)"); + shell.executeStatement("CREATE TABLE " + identifier.name() + + " STORED BY ICEBERG STORED AS PARQUET AS SELECT id FROM " + source + " WHERE id < 0"); + + // writing no rows commits no snapshot, so the table has no state for statistics to describe + Assert.assertNull(testTables.loadTable(identifier).currentSnapshot()); + Assert.assertNull(storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertFalse("no statistics stand for a table that holds nothing", + colStatsAccurate(identifier)); + } + + @Test + public void testColStatsAfterEmptyingPartitionedTable() { + // a delete of every row and a truncate both leave the table empty: the statistics of the rows + // they removed must not outlive them + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_emptied"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) PARTITIONED BY (p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2', 'external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'a\') VALUES (1), (2)"); + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'b\') VALUES (7), (9)"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 9); + + shell.executeStatement("DELETE FROM " + identifier); + Assert.assertEquals(Long.valueOf(0L), storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the walk stops at the emptied snapshot", readCurrentColStats(identifier).isEmpty()); + Assert.assertFalse(colStatsAccurate(identifier)); + + // and the next write re-anchors the chain rather than extending what the delete left + shell.executeStatement("INSERT INTO " + identifier + " PARTITION (p=\'c\') VALUES (20)"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 20, 20); + + shell.executeStatement("TRUNCATE TABLE " + identifier); + Assert.assertEquals(Long.valueOf(0L), storageHandler().getRowCount(hmsTable(identifier))); + Assert.assertTrue("the truncate leaves nothing to serve", readCurrentColStats(identifier).isEmpty()); + } + @Test public void testStatsRemoved() throws IOException { Assume.assumeTrue("Only HiveCatalog can remove stats which become obsolete", @@ -772,7 +984,7 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { // are accounted there List colStats = IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); + icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); Assert.assertEquals( List.of(DummyPartition.VOID, "last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), @@ -1051,14 +1263,14 @@ public void testAutoGatherSkipsPartitionedInsert() { @Test public void testTableLevelColStatsFallbackForPartitioned() throws Exception { - // hive.analyze.stmt.collect.partlevel.stats=false trades partition granularity for cheap + // hive.iceberg.stats.collect.partlevel=false trades partition granularity for cheap // maintenance: ANALYZE and autogather keep a single table-level file for the partitioned // table, inserts merge into it incrementally, and planning serves it over the pruned set assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "orders_tbl_level"); shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); createDatePartitionedTable(identifier); shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, date '2023-03-04'), (3, date '2023-03-04')"); Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); @@ -1330,10 +1542,15 @@ private boolean colStatsAccurate(TableIdentifier identifier) { return StatsSetupConst.areColumnStatsUptoDate(hmsTable(identifier).getParameters(), "id"); } + /** The granularity this table's statistics are written at, which is what a read asks back for. */ + private boolean partitionLevel(Table icebergTable) { + return IcebergTableUtil.isPartitionStats(icebergTable, shell.getHiveConf()); + } + private List readCurrentColStats(TableIdentifier identifier) { Table icebergTable = testTables.loadTable(identifier); return IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); + icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); } private boolean hasColStatsForCurrentSnapshot(TableIdentifier identifier) { @@ -1477,7 +1694,7 @@ private List colStatsPartNames(TableIdentifier identifier) { Table icebergTable = testTables.loadTable(identifier); List colStats = IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf(), null); + icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); return colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList(); } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q new file mode 100644 index 000000000000..e5a2cb87122e --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_granularity.q @@ -0,0 +1,37 @@ +-- SORT_QUERY_RESULTS +set hive.explain.user=false; +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.fetch.task.conversion=none; + +create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2'); + +insert into ice_p values (1,'a'),(2,'a'),(7,'b'); + +-- 1) partition-level statistics for the current snapshot +set hive.iceberg.stats.collect.partlevel=true; +analyze table ice_p compute statistics for columns; +describe formatted ice_p id; +explain select * from ice_p where p='a'; + +-- 2) a new snapshot, then table-level statistics for it +insert into ice_p values (9,'c'); +set hive.iceberg.stats.collect.partlevel=false; +analyze table ice_p compute statistics for columns; +describe formatted ice_p id; +-- table-level statistics serve the pruned set +explain select * from ice_p where p='a'; + +-- 3) toggle back: the current snapshot's file is table-level shaped, so the partition-level +-- read falls back to the older partition-level file of the ancestor snapshot +set hive.iceberg.stats.collect.partlevel=true; +explain select * from ice_p where p='a'; +describe formatted ice_p id; + +-- 4) recompute at the current snapshot with the flag on +analyze table ice_p compute statistics for columns; +explain select * from ice_p where p='a'; +describe formatted ice_p id; + +drop table ice_p; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out index dfbdfaa45270..e0767e06c45f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out @@ -35,23 +35,23 @@ Stage-0 Stage-1 Map 1 vectorized File Output Operator [FS_23] - Map Join Operator [MAPJOIN_22] (rows=2 width=607) + Map Join Operator [MAPJOIN_22] (rows=2 width=530) BucketMapJoin:true,Conds:SEL_21._col1, _col2=RS_19._col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 2 [CUSTOM_EDGE] vectorized MULTICAST [RS_19] PartitionCols:_col2, _col1 - Select Operator [SEL_18] (rows=2 width=552) + Select Operator [SEL_18] (rows=2 width=265) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_17] (rows=2 width=552) + Filter Operator [FIL_17] (rows=2 width=265) predicate:(id is not null and part is not null) - TableScan [TS_3] (rows=2 width=552) - default@tbl,tbl2,Tbl:COMPLETE,Col:NONE,Output:["foid","part","id"] - <-Select Operator [SEL_21] (rows=2 width=552) + TableScan [TS_3] (rows=2 width=265) + default@tbl,tbl2,Tbl:COMPLETE,Col:PARTIAL,Output:["foid","part","id"] + <-Select Operator [SEL_21] (rows=2 width=265) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_20] (rows=2 width=552) + Filter Operator [FIL_20] (rows=2 width=265) predicate:(id is not null and part is not null) - TableScan [TS_0] (rows=2 width=552) - default@tbl,tbl,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] + TableScan [TS_0] (rows=2 width=265) + default@tbl,tbl,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"] PREHOOK: query: SELECT * FROM tbl JOIN tbl tbl2 ON tbl.id = tbl2.id AND tbl.part = tbl2.part PREHOOK: type: QUERY diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out index a5b35e5a76fe..ef5c4410fbb6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out @@ -359,7 +359,7 @@ num_trues num_falses bit_vector HL comment Transform: identity -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}} PREHOOK: query: EXPLAIN select count(*) from src_ice t1 join tbl_ice_puffin t2 on (t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@src_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index a8b2550e29a1..4b4dd40489b9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -292,7 +292,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"c\",\"required\":false,\"type\":\"int\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index 1da70e101f39..810776ea1de3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -199,58 +199,58 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Group By Operator keys: a (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(a) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 10 @@ -262,11 +262,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 112 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 112 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -277,13 +277,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 190 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 190 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -294,23 +294,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -330,26 +330,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 209 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -359,13 +359,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1206 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1206 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 15 Reduce Operator Tree: @@ -376,11 +376,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1326 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1326 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 16 Reduce Operator Tree: @@ -391,13 +391,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1404 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 17 Reduce Operator Tree: @@ -408,16 +408,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 18 Execution mode: vectorized @@ -425,7 +425,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -444,20 +444,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1544 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 772 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 772 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 19 Execution mode: vectorized @@ -545,11 +545,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 24 Execution mode: vectorized @@ -558,31 +558,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 25 Execution mode: vectorized @@ -591,11 +591,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 26 Execution mode: vectorized @@ -604,11 +604,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -619,17 +619,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1369 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -640,11 +640,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1505 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1505 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: @@ -655,24 +655,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1359 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: @@ -683,17 +683,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1494 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized @@ -704,7 +704,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2266 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -719,13 +719,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 102 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 102 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Union 7 Vertex: Union 7 @@ -795,7 +795,7 @@ POSTHOOK: query: insert into tbl_ice_other values (10, 'ten'), (333, 'hundred') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice_other -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -817,17 +817,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -851,44 +850,44 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -909,19 +908,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -930,20 +916,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -963,44 +949,44 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -1059,29 +1045,15 @@ STAGE PLANS: expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1090,21 +1062,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) - Reducer 8 + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1123,22 +1095,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 776 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) - Reducer 9 + Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1147,54 +1120,67 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1215,7 +1201,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1259,7 +1245,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_standard_other POSTHOOK: Lineage: tbl_standard_other.a SCRIPT [] POSTHOOK: Lineage: tbl_standard_other.b SCRIPT [] -Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[175][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice @@ -1281,17 +1267,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) + Reducer 10 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) + Reducer 12 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 12 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1315,44 +1300,44 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized - Map 14 + Map 13 Map Operator Tree: TableScan alias: t2 @@ -1373,25 +1358,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 10 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) - Reducer 11 Reduce Operator Tree: Merge Join Operator condition map: @@ -1400,20 +1366,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1433,38 +1399,38 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 12 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1523,29 +1489,15 @@ STAGE PLANS: expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 646 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice Reducer 6 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.tbl_ice - Reducer 7 Reduce Operator Tree: Merge Join Operator condition map: @@ -1554,21 +1506,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) - Reducer 8 + Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1587,22 +1539,23 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 388 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) - Reducer 9 + Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 646 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.tbl_ice + Reducer 8 Reduce Operator Tree: Merge Join Operator condition map: @@ -1611,62 +1564,81 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col1 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: boolean) Union 5 Vertex: Union 5 @@ -1687,7 +1659,7 @@ STAGE PLANS: Stats Work Basic Stats Work: -Warning: Shuffle Join MERGEJOIN[155][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[153][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out index a8cc833ba713..b980414bdaa6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out @@ -306,17 +306,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -362,17 +362,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -381,10 +381,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -435,17 +435,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -475,17 +475,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -494,10 +494,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -566,17 +566,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -660,13 +660,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -675,17 +675,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -694,10 +694,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -757,17 +757,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -819,13 +819,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -834,17 +834,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -853,10 +853,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -922,17 +922,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -994,17 +994,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1013,10 +1013,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1067,17 +1067,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1107,17 +1107,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1126,10 +1126,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1191,17 +1191,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1247,17 +1247,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1266,10 +1266,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1320,17 +1320,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1360,17 +1360,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1379,10 +1379,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1442,17 +1442,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1501,17 +1501,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1520,10 +1520,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1574,17 +1574,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1617,17 +1617,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1636,10 +1636,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1690,17 +1690,17 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: @@ -1762,17 +1762,17 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -1781,10 +1781,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1840,20 +1840,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -1937,13 +1937,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1952,17 +1952,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -1971,10 +1971,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2034,20 +2034,20 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Map 5 @@ -2131,13 +2131,13 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -2146,17 +2146,17 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 4 Execution mode: vectorized @@ -2165,10 +2165,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2223,48 +2223,48 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2275,17 +2275,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -2294,10 +2294,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2309,37 +2309,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -2348,37 +2348,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -2426,48 +2426,48 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2479,19 +2479,19 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Execution mode: vectorized Reduce Operator Tree: @@ -2499,10 +2499,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2514,37 +2514,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Reducer 7 Execution mode: vectorized @@ -2553,37 +2553,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Union 6 Vertex: Union 6 @@ -2633,56 +2633,56 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: ds (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 6 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -2692,13 +2692,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 174892 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -2708,10 +2708,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2723,13 +2723,13 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 174892 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -2737,37 +2737,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Reducer 9 Execution mode: vectorized @@ -2776,37 +2776,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Union 3 Vertex: Union 3 @@ -2859,11 +2859,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -2872,17 +2872,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -2928,10 +2928,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2999,11 +2999,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3013,7 +3013,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 86000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3022,17 +3022,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 2420 Data size: 846476 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3114,10 +3114,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3183,11 +3183,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3196,17 +3196,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3268,10 +3268,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3333,11 +3333,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3346,17 +3346,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3402,10 +3402,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3456,11 +3456,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: hr is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hr (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 172000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3469,17 +3469,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3528,10 +3528,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3599,14 +3599,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 11.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 11.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3616,7 +3616,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3625,17 +3625,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3717,10 +3717,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3779,14 +3779,14 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ((UDFToDouble(hr) = 13.0D) and ds is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (UDFToDouble(hr) = 13.0D) (type: boolean) - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 180000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3796,7 +3796,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 1100 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 43000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -3805,17 +3805,17 @@ STAGE PLANS: 1 _col0 (type: string) input vertices: 1 Map 4 - Statistics: Num rows: 1210 Data size: 423238 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 3 @@ -3897,10 +3897,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3955,11 +3955,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Left Semi Join 0 to 1 @@ -3969,50 +3969,50 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Union 5 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2200 Data size: 384762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Map 3 Map Operator Tree: TableScan alias: srcpart_iceberg - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Group By Operator aggregations: max(ds) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Execution mode: vectorized Reducer 2 @@ -4022,10 +4022,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1100 Data size: 192381 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4037,37 +4037,37 @@ STAGE PLANS: aggregations: min(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Reducer 6 Execution mode: vectorized @@ -4076,37 +4076,37 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Dynamic Partitioning Event Operator Target column: ds (string) Target Input: srcpart_iceberg Partition key expr: ds - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Target Vertex: Map 1 Union 5 Vertex: Union 5 @@ -4181,11 +4181,11 @@ STAGE PLANS: TableScan alias: srcpart_iceberg filterExpr: (ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 699568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 360000 Basic stats: COMPLETE Column stats: PARTIAL Map Join Operator condition map: Inner Join 0 to 1 @@ -4194,17 +4194,17 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 - Statistics: Num rows: 2200 Data size: 769524 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 16000 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Map 3 Map Operator Tree: @@ -4263,10 +4263,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out index 36b9ab3d4dff..0ace027030fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out @@ -80,54 +80,54 @@ STAGE PLANS: TableScan alias: srcpart_date_n7 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_n3_key1_min) AND DynamicValue(RS_7_srcpart_small_n3_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_n3_key1_bloom_filter))) (type: boolean) - Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Map 4 Map Operator Tree: TableScan alias: srcpart_small_n3 filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key1 is not null (type: boolean) - Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 19 Data size: 3496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=19) - minReductionHashAggr: 0.99 + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20) + minReductionHashAggr: 0.95 mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized Reducer 2 @@ -138,17 +138,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 2090 Data size: 365523 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 126 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -157,10 +157,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -169,14 +169,14 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=19) + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, 1, expectedEntries=20) mode: final outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 696 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out new file mode 100644 index 000000000000..347519552cf2 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out @@ -0,0 +1,326 @@ +PREHOOK: query: create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_p +POSTHOOK: query: create external table ice_p (id int, p string) partitioned by spec (p) +stored by iceberg stored as parquet tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_p +PREHOOK: query: insert into ice_p values (1,'a'),(2,'a'),(7,'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_p +POSTHOOK: query: insert into ice_p values (1,'a'),(2,'a'),(7,'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_p +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 7 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_p values (9,'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_p +POSTHOOK: query: insert into ice_p values (9,'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_p +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: default@ice_p@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: default@ice_p@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 9 +num_nulls 0 +distinct_count 4 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 7 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"p\":\"true\"}} +PREHOOK: query: analyze table ice_p compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: default@ice_p +PREHOOK: Output: default@ice_p@p=a +PREHOOK: Output: default@ice_p@p=b +PREHOOK: Output: default@ice_p@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_p compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: default@ice_p +POSTHOOK: Output: default@ice_p@p=a +POSTHOOK: Output: default@ice_p@p=b +POSTHOOK: Output: default@ice_p@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select * from ice_p where p='a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_p +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select * from ice_p where p='a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_p + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: describe formatted ice_p id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_p +POSTHOOK: query: describe formatted ice_p id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_p +col_name id +data_type int +min 1 +max 9 +num_nulls 0 +distinct_count 4 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"p\":\"true\"}} +PREHOOK: query: drop table ice_p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_p +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_p +POSTHOOK: query: drop table ice_p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_p +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_p diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index a474fab5216e..1793758f29f3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -267,14 +267,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: strcol (type: string), intcol (type: int), 3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -283,17 +283,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -303,14 +303,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -374,14 +374,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: strcol (type: string), intcol (type: int), 4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -390,17 +390,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -410,14 +410,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -479,7 +479,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"intcol\":\"true\",\"pcol\":\"true\",\"strcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"strcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"int\"}]} @@ -920,32 +920,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -957,7 +957,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 3336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -970,14 +970,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1039,32 +1039,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1076,7 +1076,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 12 Data size: 4464 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 3252 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1089,14 +1089,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1158,32 +1158,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 4440 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9583333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1195,7 +1195,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 24 Data size: 8928 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 6504 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1208,14 +1208,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1277,32 +1277,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 8880 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9791667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1314,7 +1314,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 48 Data size: 17484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 13008 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1327,14 +1327,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1398,32 +1398,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 96 Data size: 17664 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9895833 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1435,7 +1435,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 96 Data size: 34596 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 96 Data size: 26208 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1448,14 +1448,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1519,32 +1519,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 192 Data size: 34560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1556,7 +1556,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 192 Data size: 67712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 192 Data size: 52416 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1569,14 +1569,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1640,7 +1640,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -2461,14 +2461,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2477,17 +2477,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2497,14 +2497,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2567,14 +2567,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2583,17 +2583,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2603,14 +2603,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2674,7 +2674,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"country\":\"true\",\"name\":\"true\",\"state\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"country\",\"required\":false,\"type\":\"string\"},{\"id\":4,\"name\":\"state\",\"required\":false,\"type\":\"string\"}]} @@ -3135,14 +3135,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 45637829068876994L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3151,17 +3151,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -3171,14 +3171,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3241,14 +3241,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3257,17 +3257,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -3277,14 +3277,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3346,7 +3346,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"long\"}]} @@ -3807,14 +3807,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3823,17 +3823,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -3843,14 +3843,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3913,14 +3913,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -3929,17 +3929,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -3949,14 +3949,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4018,7 +4018,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"double\"}]} @@ -4479,14 +4479,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -4495,17 +4495,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -4515,14 +4515,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4585,14 +4585,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -4601,17 +4601,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 2064 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -4621,14 +4621,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4690,7 +4690,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"datecol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"datecol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(10, 6)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index fc32aedc3ac5..16bc406a9464 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -395,14 +395,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -411,17 +411,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -431,14 +431,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -501,14 +501,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -517,17 +517,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -537,14 +537,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -606,7 +606,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -1069,14 +1069,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), DATE'1999-12-13' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1085,17 +1085,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1105,14 +1105,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1175,14 +1175,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), DATE'1999-12-02' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1191,17 +1191,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1211,14 +1211,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1280,7 +1280,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -1743,14 +1743,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-13' (type: date), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1759,17 +1759,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1779,14 +1779,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1849,14 +1849,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-02' (type: date), 234675894076895090L (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1865,17 +1865,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1885,14 +1885,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1954,7 +1954,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2290,14 +2290,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_truncate - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'gfhuiyoprj' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2306,17 +2306,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2326,14 +2326,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2395,7 +2395,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -2715,14 +2715,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_bucket - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'gfhutjkgkd' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2731,17 +2731,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2751,14 +2751,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2820,7 +2820,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -3041,7 +3041,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(38, 0)\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 66c45229016a..7a7e3dc091fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -174,7 +174,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 2692b6e0d623..1e128a396b6d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -57,14 +57,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -73,17 +73,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -93,14 +93,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -164,14 +164,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -180,17 +180,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -200,14 +200,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -492,32 +492,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -529,7 +529,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 2224 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -542,14 +542,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -611,32 +611,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'USA' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -648,7 +648,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -661,14 +661,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -730,32 +730,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'CA' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -767,7 +767,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 4 Data size: 1488 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1084 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -780,14 +780,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -849,32 +849,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -886,7 +886,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -899,14 +899,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -970,32 +970,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1007,7 +1007,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1020,14 +1020,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1091,32 +1091,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.95 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1128,7 +1128,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1141,14 +1141,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1467,14 +1467,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-31' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1483,17 +1483,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1503,14 +1503,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1573,14 +1573,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-26' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1589,17 +1589,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1609,14 +1609,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1921,14 +1921,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 34567890123456787L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1937,17 +1937,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1957,14 +1957,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2027,14 +2027,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2043,17 +2043,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2063,14 +2063,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2375,14 +2375,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2391,17 +2391,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2411,14 +2411,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2481,14 +2481,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2497,17 +2497,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2517,14 +2517,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2829,14 +2829,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2845,17 +2845,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.75 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2865,14 +2865,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2935,14 +2935,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2951,17 +2951,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2971,14 +2971,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 9f90a4898fc8..95fda3f4c11b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -391,14 +391,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-13' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -407,17 +407,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -427,14 +427,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -497,14 +497,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_year - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-02' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -513,17 +513,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -533,14 +533,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -602,7 +602,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"}]} @@ -1039,14 +1039,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: bigintcol (type: bigint), DATE'1999-12-13' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1055,17 +1055,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1075,14 +1075,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1145,14 +1145,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_month - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 234675894076895090L (type: bigint), DATE'1999-12-02' (type: date), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1161,17 +1161,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: date), _col6 (type: date), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1181,14 +1181,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'DATE' (type: string), _col5 (type: date), _col6 (type: date), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1250,7 +1250,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"pcol\",\"required\":false,\"type\":\"date\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} @@ -1691,14 +1691,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-13' (type: date), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1707,17 +1707,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.8333333 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1727,14 +1727,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1797,14 +1797,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: DATE'1999-12-02' (type: date), 234675894076895090L (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1813,17 +1813,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.9166667 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1833,14 +1833,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2238,14 +2238,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_truncate - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'gfhuiyoprj' (type: string), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2254,17 +2254,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2274,14 +2274,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 580 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2343,7 +2343,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigintcol\":\"true\",\"intcol\":\"true\",\"pcol\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"bigintcol\",\"required\":false,\"type\":\"long\"},{\"id\":3,\"name\":\"intcol\",\"required\":false,\"type\":\"int\"}]} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out index b87b1a62286b..8236bb2c9609 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out @@ -131,9 +131,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice_pp_key - Statistics: Num rows: 10 Data size: 63810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 63730 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 10 Data size: 63810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 63730 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.9 @@ -151,9 +151,9 @@ STAGE PLANS: TableScan alias: tbl_ice_pp_key As of version: s1 - Statistics: Num rows: 2 Data size: 12636 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 12620 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 2 Data size: 12636 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 12620 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.5 @@ -272,19 +272,19 @@ STAGE PLANS: TableScan alias: tbl_ice_pp_key filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 8 Data size: 51174 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 8 Data size: 51174 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - minReductionHashAggr: 0.875 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Map 4 @@ -293,22 +293,22 @@ STAGE PLANS: alias: tbl_ice_pp_key As of version: s1 filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Filter Operator predicate: (a > 2) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 2 @@ -318,14 +318,14 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'current' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -337,14 +337,14 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL Select Operator expressions: 'asof_s1' (type: string), _col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 107 Basic stats: PARTIAL Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out index 17e9f416d971..fd34901d33a2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -58,14 +58,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -109,14 +109,14 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out index 22498fe40b83..57bdbac57c57 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out @@ -53,15 +53,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice01 - filterExpr: ((year = 2023) and (month = 10) and (day = 3)) (type: boolean) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + filterExpr: ((day = 3) and (year = 2023) and (month = 10)) (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index de26ccd32be0..91f65d1827a1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -85,17 +85,17 @@ STAGE PLANS: TableScan alias: test_ice_int filterExpr: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 22) (type: boolean) - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -280,17 +280,17 @@ STAGE PLANS: TableScan alias: test_ice_bigint filterExpr: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 297 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 226784902765739L) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 226784902765739L (type: bigint), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -501,17 +501,17 @@ STAGE PLANS: TableScan alias: test_ice_str filterExpr: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 693 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (b = 'ddd') (type: boolean) - Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), 'ddd' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 1344 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -742,17 +742,17 @@ STAGE PLANS: TableScan alias: test_ice_date filterExpr: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (b = DATE'2022-02-07') (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), DATE'2022-02-07' (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -930,17 +930,17 @@ STAGE PLANS: TableScan alias: test_ice_double filterExpr: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a = 1.156748927566759E11D) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), b (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1111,18 +1111,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test_ice_double_date - filterExpr: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + filterExpr: ((b = DATE'2022-02-07') and (a = 1.156748927566759E11D)) (type: boolean) + Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + predicate: ((b = DATE'2022-02-07') and (a = 1.156748927566759E11D)) (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index ae9c6aa6a61f..3e3f7761f053 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -107,7 +107,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} EXTERNAL TRUE bucketing_version 2 format-version 2 @@ -362,7 +362,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} EXTERNAL TRUE bucketing_version 2 format-version 2 @@ -434,7 +434,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} EXTERNAL TRUE bucketing_version 2 format-version 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out index ee4e941ebbd0..1b75e3b7acd7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out @@ -123,13 +123,13 @@ Stage-0 Filter Operator [FIL_14] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_2] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -194,13 +194,13 @@ Stage-0 Filter Operator [FIL_18] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_2] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -257,13 +257,13 @@ Stage-0 Filter Operator [FIL_14] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_2] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -328,13 +328,13 @@ Stage-0 Filter Operator [FIL_18] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_2] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -391,13 +391,13 @@ Stage-0 Filter Operator [FIL_27] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_31] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_30] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -462,13 +462,13 @@ Stage-0 Filter Operator [FIL_41] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_48] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_47] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s @@ -525,13 +525,13 @@ Stage-0 Filter Operator [FIL_27] (rows=1 width=168) predicate:((CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_3] (rows=1 width=168) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","decimal_col"] <-Select Operator [SEL_31] (rows=1 width=168) Output:["_col0","_col1"] Filter Operator [FIL_30] (rows=1 width=252) predicate:((string_col = 'pipeline') and (CAST( decimal_col AS STRING) = '50000000000000000005905545593') and date_col is not null) TableScan [TS_0] (rows=1 width=252) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s @@ -596,13 +596,13 @@ Stage-0 Filter Operator [FIL_41] (rows=7 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_3] (rows=7 width=260) - default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@source_table,source_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] <-Select Operator [SEL_48] (rows=20 width=168) Output:["_col0","_col1"] Filter Operator [FIL_47] (rows=20 width=260) predicate:((string_col = 'pipeline') and date_col is not null and decimal_col is not null) TableScan [TS_0] (rows=20 width=260) - default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"] + default@target_table,target_table,Tbl:COMPLETE,Col:PARTIAL,Output:["date_col","string_col","decimal_col"] PREHOOK: query: select * from target_table inner join (select distinct date_col, 'pipeline' string_col, decimal_col from source_table where string_col = 'pipeline') s diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out index ecf1ece52e32..99971bfe6268 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out @@ -252,36 +252,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=704) + Select Operator [SEL_37] (rows=1 width=798) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=704) + Group By Operator [GBY_36] (rows=1 width=500) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=1045 width=178) + Select Operator [SEL_31] (rows=785 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=1045 width=178) + Map Join Operator [MAPJOIN_30] (rows=785 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=227 width=180) + Select Operator [SEL_26] (rows=238 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=227 width=180) + Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=950 width=178) + TableScan [TS_0] (rows=238 width=95) + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] + <-Select Operator [SEL_29] (rows=1000 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=950 width=178) + Filter Operator [FIL_28] (rows=1000 width=95) predicate:key is not null - TableScan [TS_3] (rows=1000 width=178) - default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=1000 width=95) + default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=1045 width=178) + Select Operator [SEL_33] (rows=785 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] @@ -416,36 +416,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=704) + Select Operator [SEL_37] (rows=1 width=798) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=704) + Group By Operator [GBY_36] (rows=1 width=500) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result"} - Select Operator [SEL_31] (rows=547 width=179) + Select Operator [SEL_31] (rows=809 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=547 width=179) + Map Join Operator [MAPJOIN_30] (rows=809 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=227 width=180) + Select Operator [SEL_26] (rows=238 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=227 width=180) + Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=498 width=179) + TableScan [TS_0] (rows=238 width=95) + default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] + <-Select Operator [SEL_29] (rows=524 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=498 width=179) + Filter Operator [FIL_28] (rows=524 width=95) predicate:key is not null - TableScan [TS_3] (rows=524 width=179) - default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=524 width=95) + default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=547 width=179) + Select Operator [SEL_33] (rows=809 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out index a344f7479452..bb6dc3571e76 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out @@ -129,23 +129,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=5 width=198) + Map Join Operator [MAPJOIN_26] (rows=372 width=8) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=5 width=180) + Select Operator [SEL_22] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=5 width=180) + Filter Operator [FIL_21] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=180) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=5 width=180) + TableScan [TS_3] (rows=238 width=89) + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=5 width=180) + Filter Operator [FIL_24] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=89) + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b @@ -203,23 +203,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_33] Group By Operator [GBY_32] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_31] (rows=5 width=198) + Map Join Operator [MAPJOIN_31] (rows=372 width=8) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=5 width=180) + Select Operator [SEL_27] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_26] (rows=5 width=180) + Filter Operator [FIL_26] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=180) - default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - <-Select Operator [SEL_30] (rows=5 width=180) + TableScan [TS_3] (rows=238 width=89) + default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_29] (rows=5 width=180) + Filter Operator [FIL_29] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=89) + default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out index 857c74642a6f..158c2ffc10b2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out @@ -242,23 +242,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=732 width=179) + Map Join Operator [MAPJOIN_26] (rows=1797 width=8) Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=666 width=179) + Select Operator [SEL_22] (rows=738 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=666 width=179) - predicate:(key is not null and part is not null) - TableScan [TS_3] (rows=738 width=179) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=666 width=179) + Filter Operator [FIL_21] (rows=738 width=89) + predicate:(part is not null and key is not null) + TableScan [TS_3] (rows=738 width=89) + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + <-Select Operator [SEL_25] (rows=738 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=666 width=179) - predicate:(key is not null and part is not null) - TableScan [TS_0] (rows=738 width=179) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + Filter Operator [FIL_24] (rows=738 width=89) + predicate:(part is not null and key is not null) + TableScan [TS_0] (rows=738 width=89) + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b @@ -309,29 +309,29 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_37] Group By Operator [GBY_36] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_35] (rows=732 width=179) + Map Join Operator [MAPJOIN_35] (rows=1797 width=8) Conds:SEL_34._col0, _col1=RS_29._col0, _col1(Inner) <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_29] PartitionCols:_col0, _col1 - Select Operator [SEL_28] (rows=666 width=179) + Select Operator [SEL_28] (rows=738 width=89) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=666 width=179) - predicate:(key is not null and part is not null) - TableScan [TS_3] (rows=738 width=179) - default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - Dynamic Partitioning Event Operator [EVENT_32] (rows=666 width=179) - Group By Operator [GBY_31] (rows=666 width=179) + Filter Operator [FIL_27] (rows=738 width=89) + predicate:(part is not null and key is not null) + TableScan [TS_3] (rows=738 width=89) + default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=85) + Group By Operator [GBY_31] (rows=2 width=85) Output:["_col0"],keys:_col0 - Select Operator [SEL_30] (rows=666 width=179) + Select Operator [SEL_30] (rows=738 width=85) Output:["_col0"] Please refer to the previous Select Operator [SEL_28] - <-Select Operator [SEL_34] (rows=666 width=179) + <-Select Operator [SEL_34] (rows=738 width=89) Output:["_col0","_col1"] - Filter Operator [FIL_33] (rows=666 width=179) - predicate:(key is not null and part is not null) - TableScan [TS_0] (rows=738 width=179) - default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:NONE,Output:["key","part"] + Filter Operator [FIL_33] (rows=738 width=89) + predicate:(part is not null and key is not null) + TableScan [TS_0] (rows=738 width=89) + default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out index 25a47b5d3522..45295e39b953 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out @@ -181,23 +181,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_33] Group By Operator [GBY_32] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_31] (rows=5 width=198) + Map Join Operator [MAPJOIN_31] (rows=372 width=8) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=5 width=180) + Select Operator [SEL_27] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_26] (rows=5 width=180) + Filter Operator [FIL_26] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=180) - default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - <-Select Operator [SEL_30] (rows=5 width=180) + TableScan [TS_3] (rows=238 width=89) + default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + <-Select Operator [SEL_30] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_29] (rows=5 width=180) + Filter Operator [FIL_29] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=89) + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b @@ -248,23 +248,23 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_28] Group By Operator [GBY_27] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_26] (rows=5 width=198) + Map Join Operator [MAPJOIN_26] (rows=372 width=8) BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner) <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_22] (rows=5 width=180) + Select Operator [SEL_22] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_21] (rows=5 width=180) + Filter Operator [FIL_21] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_3] (rows=238 width=180) - default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:NONE,Output:["key","part"] - <-Select Operator [SEL_25] (rows=5 width=180) + TableScan [TS_3] (rows=238 width=89) + default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:PARTIAL,Output:["key","part"] + <-Select Operator [SEL_25] (rows=238 width=4) Output:["_col0"] - Filter Operator [FIL_24] (rows=5 width=180) + Filter Operator [FIL_24] (rows=238 width=89) predicate:((part = '1') and key is not null) - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] + TableScan [TS_0] (rows=238 width=89) + default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"] PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*) FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out index afde412850de..153fdc311b89 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out @@ -142,36 +142,36 @@ Stage-3 Stage-1 Reducer 3 vectorized, llap File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=704) + Select Operator [SEL_37] (rows=1 width=798) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Group By Operator [GBY_36] (rows=1 width=704) + Group By Operator [GBY_36] (rows=1 width=500) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap File Output Operator [FS_32] table:{"name:":"default.bucketmapjoin_tmp_result_n3"} - Select Operator [SEL_31] (rows=547 width=179) + Select Operator [SEL_31] (rows=809 width=366) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_30] (rows=547 width=179) + Map Join Operator [MAPJOIN_30] (rows=809 width=186) BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 1 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_27] PartitionCols:_col0 - Select Operator [SEL_26] (rows=227 width=180) + Select Operator [SEL_26] (rows=238 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=227 width=180) + Filter Operator [FIL_25] (rows=238 width=95) predicate:key is not null - TableScan [TS_0] (rows=238 width=180) - default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_29] (rows=498 width=179) + TableScan [TS_0] (rows=238 width=95) + default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key","value"] + <-Select Operator [SEL_29] (rows=524 width=95) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=498 width=179) + Filter Operator [FIL_28] (rows=524 width=95) predicate:key is not null - TableScan [TS_3] (rows=524 width=179) - default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] + TableScan [TS_3] (rows=524 width=95) + default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_35] Group By Operator [GBY_34] (rows=1 width=704) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"] - Select Operator [SEL_33] (rows=547 width=179) + Select Operator [SEL_33] (rows=809 width=366) Output:["key","value1","value2"] Please refer to the previous Select Operator [SEL_31] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out index b6244c744e3d..520b925a0bf7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out @@ -41,40 +41,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_46] - Limit [LIM_45] (rows=20 width=178) + Limit [LIM_45] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_44] (rows=550 width=178) + Select Operator [SEL_44] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_43] - Top N Key Operator [TNK_42] (rows=550 width=178) + Top N Key Operator [TNK_42] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_41] (rows=550 width=178) - Conds:RS_38._col0, _col1=SEL_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_38] + Map Join Operator [MAPJOIN_41] (rows=791 width=447) + BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [CUSTOM_EDGE] vectorized, llap + MULTICAST [RS_38] PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=450 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_36] (rows=450 width=525) - predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_40] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_39] (rows=500 width=178) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Select Operator [SEL_37] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_36] (rows=500 width=178) + predicate:(key is not null and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_40] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_39] (rows=500 width=269) + predicate:(key1 is not null and key2 is not null) + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -139,40 +139,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_46] - Limit [LIM_45] (rows=20 width=178) + Limit [LIM_45] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_44] (rows=550 width=178) + Select Operator [SEL_44] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_43] - Top N Key Operator [TNK_42] (rows=550 width=178) + Top N Key Operator [TNK_42] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_41] (rows=550 width=178) - Conds:RS_38._col0, _col1=SEL_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_38] + Map Join Operator [MAPJOIN_41] (rows=791 width=447) + BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [CUSTOM_EDGE] vectorized, llap + MULTICAST [RS_38] PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=475 width=778) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_36] (rows=475 width=778) - predicate:(key2 is not null and (key1 <> '0') and (key1 <> '100')) - TableScan [TS_0] (rows=500 width=778) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_40] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_39] (rows=500 width=178) - predicate:((key <> '0') and (key <> '100') and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Select Operator [SEL_37] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_36] (rows=500 width=178) + predicate:((key <> '0') and (key <> '100') and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_40] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_39] (rows=500 width=269) + predicate:((key1 <> '0') and (key1 <> '100') and key2 is not null) + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -237,40 +237,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=178) + Limit [LIM_35] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_34] (rows=550 width=178) + Select Operator [SEL_34] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=550 width=178) + Top N Key Operator [TNK_32] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=550 width=178) - Conds:RS_28._col0=SEL_30._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_28] + Map Join Operator [MAPJOIN_31] (rows=791 width=447) + BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [CUSTOM_EDGE] vectorized, llap + MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=475 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_26] (rows=475 width=525) - predicate:key1 is not null - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_30] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_29] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Select Operator [SEL_27] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_30] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_29] (rows=500 width=269) + predicate:key1 is not null + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -344,15 +344,15 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=778) + Limit [LIM_35] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_34] (rows=550 width=778) + Select Operator [SEL_34] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=550 width=778) + Top N Key Operator [TNK_32] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=550 width=778) + Map Join Operator [MAPJOIN_31] (rows=791 width=447) BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_28] @@ -363,12 +363,12 @@ Stage-0 predicate:((key <> '0') and (key <> '100')) TableScan [TS_3] (rows=500 width=178) default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_30] (rows=500 width=778) + <-Select Operator [SEL_30] (rows=500 width=269) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=500 width=778) + Filter Operator [FIL_29] (rows=500 width=269) predicate:((key1 <> '0') and (key1 <> '100')) - TableScan [TS_0] (rows=500 width=778) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -435,40 +435,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=178) + Limit [LIM_35] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_34] (rows=550 width=178) + Select Operator [SEL_34] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=550 width=178) + Top N Key Operator [TNK_32] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=550 width=178) - Conds:RS_28._col0=SEL_30._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_28] + Map Join Operator [MAPJOIN_31] (rows=791 width=447) + BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [CUSTOM_EDGE] vectorized, llap + MULTICAST [RS_28] PartitionCols:_col0 - Select Operator [SEL_27] (rows=475 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_26] (rows=475 width=525) - predicate:(key1 is not null and (key2 <> 'val_0') and (key2 <> 'val_100')) - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_30] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_29] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Select Operator [SEL_27] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=500 width=178) + predicate:key is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_30] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_29] (rows=500 width=269) + predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null) + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -533,40 +533,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_36] - Limit [LIM_35] (rows=20 width=178) + Limit [LIM_35] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_34] (rows=550 width=178) + Select Operator [SEL_34] (rows=814 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_33] - Top N Key Operator [TNK_32] (rows=550 width=178) + Top N Key Operator [TNK_32] (rows=814 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_31] (rows=550 width=178) - Conds:RS_28._col1=SEL_30._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap + Map Join Operator [MAPJOIN_31] (rows=814 width=447) + Conds:SEL_30._col1=RS_28._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_28] PartitionCols:_col1 - Select Operator [SEL_27] (rows=475 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_26] (rows=475 width=525) - predicate:key2 is not null - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_30] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_29] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Select Operator [SEL_27] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=500 width=178) + predicate:value is not null + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_30] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_29] (rows=500 width=269) + predicate:key2 is not null + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -629,40 +629,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (CUSTOM_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_51] - Limit [LIM_50] (rows=20 width=178) + Limit [LIM_50] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_49] (rows=550 width=178) + Select Operator [SEL_49] (rows=791 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_48] - Top N Key Operator [TNK_47] (rows=550 width=178) + Top N Key Operator [TNK_47] (rows=791 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_46] (rows=550 width=178) - Conds:RS_43._col0, _col1, _col2=SEL_45._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap - BROADCAST [RS_43] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_42] (rows=425 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_41] (rows=425 width=525) - predicate:(key1 is not null and key2 is not null and value is not null) - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_45] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_44] (rows=500 width=178) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Map Join Operator [MAPJOIN_46] (rows=791 width=447) + BucketMapJoin:true,Conds:SEL_45._col0, _col1, _col2=RS_43._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [CUSTOM_EDGE] vectorized, llap + MULTICAST [RS_43] + PartitionCols:_col0, _col1 + Select Operator [SEL_42] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_41] (rows=500 width=178) + predicate:(key is not null and value is not null) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_45] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_44] (rows=500 width=269) + predicate:(key1 is not null and key2 is not null and value is not null) + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a @@ -725,40 +725,40 @@ POSTHOOK: Input: default@srcbucket_big Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:20 Stage-1 - Reducer 3 vectorized, llap + Reducer 2 vectorized, llap File Output Operator [FS_41] - Limit [LIM_40] (rows=20 width=178) + Limit [LIM_40] (rows=20 width=447) Number of rows:20 - Select Operator [SEL_39] (rows=550 width=178) + Select Operator [SEL_39] (rows=814 width=447) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 2 [SIMPLE_EDGE] vectorized, llap + <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_38] - Top N Key Operator [TNK_37] (rows=550 width=178) + Top N Key Operator [TNK_37] (rows=814 width=447) keys:_col0,top n:20 - Map Join Operator [MAPJOIN_36] (rows=550 width=178) - Conds:RS_33._col2=SEL_35._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [BROADCAST_EDGE] vectorized, llap + Map Join Operator [MAPJOIN_36] (rows=814 width=447) + Conds:SEL_35._col2=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_33] - PartitionCols:_col2 - Select Operator [SEL_32] (rows=475 width=525) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=475 width=525) + PartitionCols:_col1 + Select Operator [SEL_32] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_31] (rows=500 width=178) predicate:value is not null - TableScan [TS_0] (rows=500 width=525) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["key1","key2","value"] - <-Select Operator [SEL_35] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_34] (rows=500 width=178) + TableScan [TS_3] (rows=500 width=178) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Select Operator [SEL_35] (rows=500 width=269) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_34] (rows=500 width=269) predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + TableScan [TS_0] (rows=500 width=269) + default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] PREHOOK: query: SELECT * FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out index 2c4de1e43e85..0afae5cee90d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out @@ -148,11 +148,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=17 width=8) + Select Operator [SEL_31] (rows=11 width=7) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=17 width=8) + Map Join Operator [MAPJOIN_29] (rows=11 width=7) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -163,12 +163,12 @@ Stage-0 predicate:key1 is not null TableScan [TS_3] (rows=6 width=3) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1"] - <-Select Operator [SEL_28] (rows=16 width=8) + <-Select Operator [SEL_28] (rows=11 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=8) + Filter Operator [FIL_27] (rows=11 width=7) predicate:key1 is not null - TableScan [TS_0] (rows=16 width=8) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] + TableScan [TS_0] (rows=16 width=7) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"] PREHOOK: query: SELECT a.key1, a.id FROM srcbucket_big a @@ -232,11 +232,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_32] - Select Operator [SEL_31] (rows=17 width=194) + Select Operator [SEL_31] (rows=11 width=75) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_30] - Map Join Operator [MAPJOIN_29] (rows=17 width=194) + Map Join Operator [MAPJOIN_29] (rows=11 width=75) BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_26] @@ -247,12 +247,12 @@ Stage-0 predicate:key2 is not null TableScan [TS_3] (rows=6 width=72) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key2"] - <-Select Operator [SEL_28] (rows=16 width=188) + <-Select Operator [SEL_28] (rows=11 width=75) Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=16 width=188) + Filter Operator [FIL_27] (rows=11 width=75) predicate:key2 is not null - TableScan [TS_0] (rows=16 width=188) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] + TableScan [TS_0] (rows=16 width=69) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"] PREHOOK: query: SELECT a.key2, a.id FROM srcbucket_big a @@ -316,11 +316,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_37] - Select Operator [SEL_36] (rows=22 width=188) + Select Operator [SEL_36] (rows=16 width=76) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_35] - Map Join Operator [MAPJOIN_34] (rows=22 width=188) + Map Join Operator [MAPJOIN_34] (rows=16 width=76) Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1"] <-Map 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_31] @@ -331,12 +331,12 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=6 width=74) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Select Operator [SEL_33] (rows=20 width=188) + <-Select Operator [SEL_33] (rows=16 width=76) Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=20 width=188) + Filter Operator [FIL_32] (rows=16 width=76) predicate:value is not null - TableScan [TS_0] (rows=21 width=188) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Output:["value","id"] + TableScan [TS_0] (rows=21 width=76) + default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["value","id"] PREHOOK: query: SELECT a.value, a.id FROM srcbucket_big a @@ -400,11 +400,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_42] - Select Operator [SEL_41] (rows=13 width=194) + Select Operator [SEL_41] (rows=3 width=64) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_40] - Map Join Operator [MAPJOIN_39] (rows=13 width=194) + Map Join Operator [MAPJOIN_39] (rows=3 width=64) BucketMapJoin:true,Conds:SEL_38._col0, _col1=RS_36._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_36] @@ -415,12 +415,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null) TableScan [TS_3] (rows=6 width=75) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2"] - <-Select Operator [SEL_38] (rows=12 width=192) + <-Select Operator [SEL_38] (rows=4 width=72) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=12 width=192) + Filter Operator [FIL_37] (rows=4 width=72) predicate:(key1 is not null and key2 is not null) - TableScan [TS_0] (rows=12 width=192) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] + TableScan [TS_0] (rows=12 width=64) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"] PREHOOK: query: SELECT a.key1, a.key2, a.id FROM srcbucket_big a @@ -480,11 +480,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_47] - Select Operator [SEL_46] (rows=17 width=198) + Select Operator [SEL_46] (rows=6 width=66) Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_45] - Map Join Operator [MAPJOIN_44] (rows=17 width=198) + Map Join Operator [MAPJOIN_44] (rows=6 width=66) BucketMapJoin:true,Conds:SEL_43._col0, _col1=RS_41._col0, _col1(Inner),Output:["_col0","_col1","_col2"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_41] @@ -495,12 +495,12 @@ Stage-0 predicate:(key1 is not null and value is not null) TableScan [TS_3] (rows=6 width=77) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","value"] - <-Select Operator [SEL_43] (rows=16 width=192) + <-Select Operator [SEL_43] (rows=8 width=73) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=16 width=192) + Filter Operator [FIL_42] (rows=8 width=73) predicate:(key1 is not null and value is not null) - TableScan [TS_0] (rows=16 width=192) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] + TableScan [TS_0] (rows=16 width=73) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"] PREHOOK: query: SELECT a.key1, a.value, a.id FROM srcbucket_big a @@ -560,11 +560,11 @@ Stage-0 Stage-1 Reducer 2 vectorized, llap File Output Operator [FS_52] - Select Operator [SEL_51] (rows=13 width=381) + Select Operator [SEL_51] (rows=2 width=184) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized, llap SHUFFLE [RS_50] - Map Join Operator [MAPJOIN_49] (rows=13 width=381) + Map Join Operator [MAPJOIN_49] (rows=2 width=184) BucketMapJoin:true,Conds:SEL_48._col0, _col1, _col2=RS_46._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"] <-Map 3 [CUSTOM_EDGE] vectorized, llap MULTICAST [RS_46] @@ -575,12 +575,12 @@ Stage-0 predicate:(key1 is not null and key2 is not null and value is not null) TableScan [TS_3] (rows=6 width=150) default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"] - <-Select Operator [SEL_48] (rows=12 width=376) + <-Select Operator [SEL_48] (rows=2 width=184) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=12 width=376) + Filter Operator [FIL_47] (rows=2 width=184) predicate:(key1 is not null and key2 is not null and value is not null) - TableScan [TS_0] (rows=12 width=376) - default@srcbucket_big,a,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] + TableScan [TS_0] (rows=12 width=124) + default@srcbucket_big,a,Tbl:COMPLETE,Col:PARTIAL,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"] PREHOOK: query: SELECT a.key1, a.key2, a.value, a.id FROM srcbucket_big a diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8c602ddb1975..c90b08189682 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -383,14 +383,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v1 Snapshot ref: tag_v1 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -512,14 +512,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v4 Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -554,14 +554,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -598,17 +598,17 @@ STAGE PLANS: alias: ice_orc.tag_v1 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v1 - Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -739,17 +739,17 @@ STAGE PLANS: alias: ice_orc.tag_v4 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -785,17 +785,17 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -965,14 +965,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1008,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index f97e4c10212a..b59449ea645f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -148,7 +148,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 @@ -296,7 +296,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index 2fb957b5319f..add0220e2470 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -123,7 +123,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} EXTERNAL TRUE bucketing_version 2 compactor.threshold.target.size 1500 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index d140bbc30aa7..3bb172c0df60 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -82,7 +82,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"key\":\"true\",\"value\":\"true\"}} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 1 @@ -198,7 +198,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"key\":\"true\",\"value\":\"true\"}} EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out index c41f2313aacb..cefa4effc895 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out @@ -359,16 +359,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: o - filterExpr: (itemid is not null and (((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70))) (type: boolean) - probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_29_container, bigKeyColName:itemid, smallTablePos:1, keyRatio:1.0476190476190477 - Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: NONE + filterExpr: ((((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70)) and itemid is not null) (type: boolean) + probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_29_container, bigKeyColName:itemid, smallTablePos:1, keyRatio:0.9523809523809523 + Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (itemid is not null and (((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70))) (type: boolean) - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE + predicate: ((((quantity > 0) and (quantity < 39)) or ((quantity > 39) and (quantity < 69)) or (quantity > 70)) and itemid is not null) (type: boolean) + Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: quantity (type: int), itemid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 @@ -378,17 +378,17 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 3 - Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.95238096 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) @@ -420,10 +420,10 @@ STAGE PLANS: aggregations: sum(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index 9a99f74574ff..05342b5039c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -132,68 +132,68 @@ STAGE PLANS: TableScan alias: target_ice filterExpr: (a <= 100) (type: boolean) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 100) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Execution mode: vectorized Map 12 Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col3 (type: int) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: FILE__PATH (type: string), a (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: string) Execution mode: vectorized Reducer 10 @@ -202,7 +202,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -221,20 +221,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: @@ -245,17 +245,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 4 Execution mode: vectorized @@ -266,7 +266,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -281,20 +281,20 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (_col4 is null or (_col4 <> _col7) or _col7 is null) (type: boolean) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col4 <> _col7) or _col4 is null or _col7 is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: @@ -305,13 +305,13 @@ STAGE PLANS: 0 _col2 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 14 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 2096 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Reduce Operator Tree: @@ -322,20 +322,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col1 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -355,26 +355,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: PARTIAL Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -384,13 +384,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col3 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -470,17 +470,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Execution mode: vectorized Map 4 @@ -509,20 +509,20 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 1216 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 912 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), _col6 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Execution mode: vectorized @@ -533,7 +533,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index 21800f645f62..c2529066adac 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -96,20 +96,20 @@ STAGE PLANS: TableScan alias: target_ice filterExpr: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 @@ -121,21 +121,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -143,14 +143,14 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -158,52 +158,52 @@ STAGE PLANS: name: default.target_ice Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: _col9 is null (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col0, 16) (type: int), iceberg_truncate(_col1, 3) (type: string) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator predicate: (_col9 = _col1) (type: boolean) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) outputColumnNames: _col2, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz sort order: ++++ Map-reduce partition columns: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col4 (type: bigint) Reducer 3 Execution mode: vectorized @@ -214,7 +214,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -229,7 +229,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -243,17 +243,17 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out index 8b7776d7bae8..9f1607322789 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out @@ -404,6 +404,10 @@ hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### PREHOOK: query: select file from default.ice_meta_3.metadata_log_entries PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_3 @@ -704,6 +708,10 @@ hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### +hdfs://### HDFS PATH ### PREHOOK: query: select file from default.ice_meta_3.metadata_log_entries PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 6b157cd74b40..556df48415c2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -100,7 +100,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"itemid\":\"true\",\"orderid\":\"true\",\"p1\":\"true\",\"p2\":\"true\",\"quantity\":\"true\",\"tradets\":\"true\"}} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"orderid\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"quantity\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"itemid\",\"required\":false,\"type\":\"int\"},{\"id\":4,\"name\":\"tradets\",\"required\":false,\"type\":\"timestamp\"},{\"id\":5,\"name\":\"p1\",\"required\":false,\"type\":\"string\"},{\"id\":6,\"name\":\"p2\",\"required\":false,\"type\":\"string\"}]} @@ -284,18 +284,18 @@ Stage-3 <-Reducer 2 [SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_21] PartitionCols:_col4, _col5 - Select Operator [SEL_20] (rows=11 width=420) + Select Operator [SEL_20] (rows=11 width=224) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_19] (rows=11 width=420) + Limit [LIM_19] (rows=11 width=224) Number of rows:100000 <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_18] - Select Operator [SEL_17] (rows=11 width=420) + Select Operator [SEL_17] (rows=11 width=224) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_16] (rows=11 width=420) + Limit [LIM_16] (rows=11 width=224) Number of rows:100000 - TableScan [TS_0] (rows=11 width=420) - default@llap_orders,llap_orders,Tbl:COMPLETE,Col:NONE,Output:["orderid","quantity","itemid","tradets","p1","p2"] + TableScan [TS_0] (rows=11 width=224) + default@llap_orders,llap_orders,Tbl:COMPLETE,Col:COMPLETE,Output:["orderid","quantity","itemid","tradets","p1","p2"] Reducer 4 vectorized File Output Operator [FS_29] Select Operator [SEL_28] (rows=1 width=1###) @@ -306,7 +306,7 @@ Stage-3 PARTITION_ONLY_SHUFFLE [RS_24] Group By Operator [GBY_23] (rows=1 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"] - Select Operator [SEL_22] (rows=11 width=420) + Select Operator [SEL_22] (rows=11 width=224) Output:["orderid","quantity","itemid","tradets","p1","p2"] Please refer to the previous Select Operator [SEL_20] diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 522c86d5bf95..0db0f72d011d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -51,20 +51,20 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 303 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Map 4 @@ -93,26 +93,26 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: FILE__PATH (type: string) null sort order: a sort order: + Map-reduce partition columns: FILE__PATH (type: string) - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: vectorized Reducer 3 Execution mode: vectorized @@ -123,7 +123,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -138,17 +138,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized @@ -156,7 +156,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -175,20 +175,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 576 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 5 Data size: 1084 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized @@ -196,7 +196,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col5 - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -216,26 +216,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col5 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Union 2 Vertex: Union 2 @@ -347,45 +347,45 @@ STAGE PLANS: TableScan alias: tbl_ice filterExpr: (a <= 5) (type: boolean) - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: a (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Map 26 @@ -483,11 +483,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -498,13 +498,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -515,20 +515,20 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 13 Reduce Operator Tree: @@ -554,17 +554,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -575,11 +575,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 16 Reduce Operator Tree: @@ -590,24 +590,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2447 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2447 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1631 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 17 Reduce Operator Tree: @@ -618,11 +618,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1989 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1989 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 18 Reduce Operator Tree: @@ -633,13 +633,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2106 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2106 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 19 Reduce Operator Tree: @@ -650,16 +650,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 2 Execution mode: vectorized @@ -668,38 +668,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 20 Execution mode: vectorized @@ -707,7 +707,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -726,20 +726,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2316 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1029 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1029 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 21 Execution mode: vectorized @@ -748,21 +748,21 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Reducer 22 Reduce Operator Tree: @@ -773,13 +773,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 23 Reduce Operator Tree: @@ -790,13 +790,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1809 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1809 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 24 Reduce Operator Tree: @@ -807,13 +807,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 153 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 25 Execution mode: vectorized @@ -822,11 +822,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 27 Execution mode: vectorized @@ -913,11 +913,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 168 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -928,13 +928,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 285 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -945,23 +945,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -981,26 +981,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 139 Basic stats: COMPLETE Column stats: NONE Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1010,17 +1010,17 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1794 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1794 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Execution mode: vectorized @@ -1031,7 +1031,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 3136 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1189,52 +1189,52 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1244,11 +1244,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1259,20 +1259,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1292,26 +1292,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1321,13 +1321,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized @@ -1335,7 +1335,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1354,20 +1354,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1552 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1584 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: @@ -1378,60 +1378,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1441,17 +1441,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2799 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized @@ -1462,7 +1462,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1475,17 +1475,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -1549,7 +1549,7 @@ STAGE PLANS: null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6641 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 @@ -1700,52 +1700,52 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1755,11 +1755,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -1770,20 +1770,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1803,26 +1803,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1656 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: PARTIAL Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -1832,13 +1832,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized @@ -1846,7 +1846,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -1865,20 +1865,20 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3492 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1552 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1584 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: @@ -1889,60 +1889,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: int) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1952,17 +1952,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized @@ -1973,7 +1973,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1986,17 +1986,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -2060,7 +2060,7 @@ STAGE PLANS: null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 6614 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out index a33dcfab92b3..ccf73acb8669 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out index 66273d7216bb..0a245752e303 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out index 3e6f66ad196b..fe5345d98e33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index bcb446bbfdb5..3c43b9ec35f6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -325,17 +325,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (CUSTOM_EDGE) + Map 1 <- Map 4 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: s - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] @@ -347,7 +346,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -371,8 +370,8 @@ STAGE PLANS: hashTableImplementationType: OPTIMIZED outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50 input vertices: - 1 Map 5 - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + 1 Map 4 + Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col39 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col32 (type: int), _col38 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col37 (type: int), _col6 (type: int), _col49 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col33 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col46 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col34 (type: int), _col30 (type: int), _col40 (type: decimal(7,2)), _col35 (type: int), _col4 (type: int), _col47 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col42 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col28 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col29 (type: int), _col41 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col50 (type: decimal(7,2)), _col31 (type: int), _col36 (type: int), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48 @@ -380,14 +379,14 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [49, 20, 42, 48, 34, 22, 36, 8, 4, 11, 58, 47, 7, 59, 12, 43, 18, 15, 54, 56, 21, 9, 44, 40, 50, 45, 5, 57, 6, 14, 13, 53, 52, 2, 17, 38, 3, 16, 19, 35, 10, 39, 51, 1, 37, 60, 41, 46, 55] - Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint), 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), null (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 @@ -396,13 +395,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [34, 35, 36, 37, 30, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 61, 53, 54, 55, 56, 57, 58, 59, 60] selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 61:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -414,7 +413,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -423,28 +422,25 @@ STAGE PLANS: native: true projectedOutputColumnNums: [31, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 62, 53, 54, 55, 56, 57, 58, 59, 60] selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 62:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator - keyColumns: 41:int, 64:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 31:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 62:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.store_sales Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 40:int), SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 38:int)) predicate: (_col23 is null and _col46 is null and _col35 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2451181 (type: int), _col43 (type: int), _col33 (type: int), _col36 (type: int), _col8 (type: int), _col26 (type: int), _col28 (type: int), _col12 (type: int), _col7 (type: int), _col21 (type: int), _col40 (type: int), _col9 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col29 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col37 (type: decimal(7,2)), _col34 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -453,7 +449,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [63, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 63:int - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) null sort order: zz @@ -461,12 +457,12 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 65:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int + keyColumns: 3:int, 64:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 63:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: @@ -474,7 +470,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col4 (type: int), _col6 (type: string), _col39 (type: bigint), _col44 (type: bigint) outputColumnNames: _col4, _col6, _col39, _col44 @@ -482,7 +478,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [34, 36, 35, 37] - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() Group By Vectorization: @@ -494,10 +490,10 @@ STAGE PLANS: vectorProcessingMode: HASH projectedOutputColumnNums: [0] keys: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint) - minReductionHashAggr: 0.99 + minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz @@ -509,7 +505,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 4:bigint - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint) Execution mode: vectorized Map Vectorization: @@ -526,13 +522,13 @@ STAGE PLANS: includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] - Map 5 + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint] + Map 4 Map Operator Tree: TableScan alias: store_sales filterExpr: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] @@ -542,7 +538,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongScalar(col 0:int, val 2451181), SelectColumnIsNotNull(col 2:int), SelectColumnIsNotNull(col 3:int)) predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 @@ -551,7 +547,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [23, 24, 25, 26, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col6 (type: int), _col7 (type: int) null sort order: zz @@ -564,7 +560,7 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: @@ -612,48 +608,13 @@ STAGE PLANS: File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.store_sales Reducer 3 - Execution mode: vectorized - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 25 - dataColumns: KEY._col3:int, KEY.iceberg_bucket(_col2, 3):int, VALUE._col0:int, VALUE._col1:int, VALUE._col2:int, VALUE._col3:int, VALUE._col4:int, VALUE._col5:int, VALUE._col6:int, VALUE._col7:int, VALUE._col8:int, VALUE._col9:int, VALUE._col10:int, VALUE._col11:decimal(7,2)/DECIMAL_64, VALUE._col12:decimal(7,2)/DECIMAL_64, VALUE._col13:decimal(7,2)/DECIMAL_64, VALUE._col14:decimal(7,2)/DECIMAL_64, VALUE._col15:decimal(7,2)/DECIMAL_64, VALUE._col16:decimal(7,2)/DECIMAL_64, VALUE._col17:decimal(7,2)/DECIMAL_64, VALUE._col18:decimal(7,2)/DECIMAL_64, VALUE._col19:decimal(7,2)/DECIMAL_64, VALUE._col20:decimal(7,2)/DECIMAL_64, VALUE._col21:decimal(7,2)/DECIMAL_64, VALUE._col22:decimal(7,2)/DECIMAL_64 - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: int), VALUE._col7 (type: int), VALUE._col8 (type: int), VALUE._col9 (type: int), VALUE._col10 (type: int), VALUE._col11 (type: decimal(7,2)), VALUE._col12 (type: decimal(7,2)), VALUE._col13 (type: decimal(7,2)), VALUE._col14 (type: decimal(7,2)), VALUE._col15 (type: decimal(7,2)), VALUE._col16 (type: decimal(7,2)), VALUE._col17 (type: decimal(7,2)), VALUE._col18 (type: decimal(7,2)), VALUE._col19 (type: decimal(7,2)), VALUE._col20 (type: decimal(7,2)), VALUE._col21 (type: decimal(7,2)), VALUE._col22 (type: decimal(7,2)), KEY._col3 (type: int), KEY.iceberg_bucket(_col2, 3) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col3, iceberg_bucket(_col2, 3) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1] - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.store_sales - Reducer 4 Execution mode: vectorized Reduce Vectorization: enabled: true @@ -682,14 +643,14 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterLongColGreaterLongScalar(col 4:bigint, val 1) predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -698,13 +659,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [5] selectExpressions: VectorUDFAdaptor(cardinality_violation(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -885,10 +846,9 @@ POSTHOOK: Output: default@merge_tmp_table POSTHOOK: Output: default@store_sales POSTHOOK: Output: default@store_sales Vertex dependency in root stage -Map 1 <- Map 5 (CUSTOM_EDGE) +Map 1 <- Map 4 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) -Reducer 4 <- Map 1 (SIMPLE_EDGE) Stage-6 Stats Work{} @@ -899,66 +859,53 @@ Stage-6 Dependency Collection{} Stage-4 Reducer 2 vectorized - File Output Operator [FS_59] + File Output Operator [FS_57] table:{"name:":"default.store_sales"} - Select Operator [SEL_58] + Select Operator [SEL_56] Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_54] + SHUFFLE [RS_53] PartitionCols:_col3, iceberg_bucket(_col2, 3) - Select Operator [SEL_50] (rows=1 width=#Masked#) + Select Operator [SEL_49] (rows=3 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - Filter Operator [FIL_46] (rows=1 width=#Masked#) - predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) - Select Operator [SEL_44] (rows=5 width=#Masked#) + Filter Operator [FIL_45] (rows=3 width=#Masked#) + predicate:(_col23 is null and _col46 is null and _col35 is null) + Select Operator [SEL_42] (rows=7 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48"] - Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) - BucketMapJoin:true,Conds:SEL_42._col1, _col2=RS_41._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] - <-Map 5 [CUSTOM_EDGE] vectorized - MULTICAST [RS_41] + Map Join Operator [MAPJOIN_41] (rows=7 width=#Masked#) + BucketMapJoin:true,Conds:SEL_40._col1, _col2=RS_39._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] + <-Map 4 [CUSTOM_EDGE] vectorized + MULTICAST [RS_39] PartitionCols:_col6 - Select Operator [SEL_40] (rows=2 width=#Masked#) + Select Operator [SEL_38] (rows=2 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"] - Filter Operator [FIL_39] (rows=2 width=#Masked#) + Filter Operator [FIL_37] (rows=2 width=#Masked#) predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) TableScan [TS_2] (rows=2 width=#Masked#) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] - <-Select Operator [SEL_42] (rows=5 width=#Masked#) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + <-Select Operator [SEL_40] (rows=5 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] TableScan [TS_0] (rows=5 width=#Masked#) - default@ssv,s,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + default@ssv,s,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] Reducer 3 vectorized File Output Operator [FS_61] - table:{"name:":"default.store_sales"} - Select Operator [SEL_60] - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_55] - PartitionCols:_col3, iceberg_bucket(_col2, 3) - Select Operator [SEL_51] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - Filter Operator [FIL_47] (rows=1 width=#Masked#) - predicate:(_col23 is null and _col46 is null and _col35 is null) - Please refer to the previous Select Operator [SEL_44] - Reducer 4 vectorized - File Output Operator [FS_65] table:{"name:":"default.merge_tmp_table"} - Select Operator [SEL_64] (rows=1 width=#Masked#) + Select Operator [SEL_60] (rows=1 width=#Masked#) Output:["_col0"] - Filter Operator [FIL_63] (rows=1 width=#Masked#) + Filter Operator [FIL_59] (rows=1 width=#Masked#) predicate:(_col4 > 1L) - Group By Operator [GBY_62] (rows=1 width=#Masked#) + Group By Operator [GBY_58] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_57] + SHUFFLE [RS_55] PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_56] (rows=1 width=#Masked#) + Group By Operator [GBY_54] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col39, _col6, _col44 - Select Operator [SEL_52] (rows=1 width=#Masked#) + Select Operator [SEL_50] (rows=1 width=#Masked#) Output:["_col4","_col6","_col39","_col44"] - Filter Operator [FIL_48] (rows=1 width=#Masked#) + Filter Operator [FIL_46] (rows=1 width=#Masked#) predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) - Please refer to the previous Select Operator [SEL_44] + Please refer to the previous Select Operator [SEL_42] Stage-7 Stats Work{} Stage-3 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java index 10f90c55e8ea..6c820d303f25 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java @@ -22,9 +22,9 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.session.SessionState; + +import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getQualifiedTableName; /** * Utilities for semantic analyzers. @@ -47,10 +47,9 @@ public static boolean isPartitionLevelStats(ASTNode tree) { } public static Table getTable(ASTNode tree, BaseSemanticAnalyzer sa) throws SemanticException { - String tableName = ColumnStatsSemanticAnalyzer.getUnescapedName((ASTNode) tree.getChild(0).getChild(0)); - String currentDb = SessionState.get().getCurrentDatabase(); - String [] names = Utilities.getDbTableName(currentDb, tableName); - return sa.getTable(names[0], names[1], true); + return sa.getTable( + getQualifiedTableName((ASTNode) tree.getChild(0).getChild(0)), + true); } public static Map getPartKeyValuePairsFromAST(Table tbl, ASTNode tree, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 33f19de294c3..601c346125e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -324,6 +324,10 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, } else { rewrittenQueryBuilder.append(unparseIdentifier(tbl.getDbName(), conf)) .append(".").append(unparseIdentifier(tbl.getTableName(), conf)); + + if (tbl.getSnapshotRef() != null) { + rewrittenQueryBuilder.append(".").append(unparseIdentifier(tbl.getSnapshotRef(), conf)); + } } // If partition level statistics is requested, add predicate and group by as needed to rewritten @@ -607,6 +611,10 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { */ if (shouldRewrite(ast)) { tbl = AnalyzeCommandUtils.getTable(ast, this); + + if (tbl.getMetaTable() != null) { + throw new SemanticException("Cannot analyze metadata table: " + tbl.getMetaTable()); + } originalTree = ast; boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) || StatsUtils.isPartitionStats(tbl, conf); @@ -719,7 +727,7 @@ AnalyzeRewriteContext getAnalyzeRewriteContext() { static AnalyzeRewriteContext genAnalyzeRewriteContext(HiveConf conf, Table tbl) { AnalyzeRewriteContext analyzeRewrite = new AnalyzeRewriteContext(); analyzeRewrite.setTableName(tbl.getFullyQualifiedName()); - analyzeRewrite.setTblLvl(!(conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && tbl.isPartitioned())); + analyzeRewrite.setTblLvl(!StatsUtils.isPartitionStats(tbl, conf)); analyzeRewrite.setFieldSchemas(getStatsEligibleFieldSchemas(tbl)); return analyzeRewrite; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java index 53b9af9384a1..1f8fe6736b42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java @@ -464,8 +464,11 @@ private int updatePartitions(Hive db, List scs, Table table) thro } if (values.get(0).result instanceof Table) { - db.alterTable(tableFullName, (Table) values.get(0).result, environmentContext, true); - LOG.debug("Updated stats for {}.", tableFullName); + // the metastore keeps one set of counts, and they describe the table, not a branch + if (table.getSnapshotRef() == null) { + db.alterTable(tableFullName, (Table) values.get(0).result, environmentContext, true); + LOG.debug("Updated stats for {}.", tableFullName); + } } else { if (values.get(0).result instanceof Partition) { List results = Lists.transform(values, StatCollector.EXTRACT_RESULT_FUNCTION); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 239105fad78d..fa244289832f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -328,7 +328,10 @@ private int aggregateStats(Hive db, Table tbl) { if (res == null) { return 0; } - db.alterTable(tableFullName, res, environmentContext, true); + // the metastore keeps one set of counts, and they describe the table, not a branch + if (table.getSnapshotRef() == null) { + db.alterTable(tableFullName, res, environmentContext, true); + } TransactionalStatsProcessor transactionalStatsProcessor = new TransactionalStatsProcessor(db, p); transactionalStatsProcessor.process(statsAggregator); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 15d23a8e329b..15453a6ec385 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -207,7 +207,7 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce boolean done = false; boolean useStorageHandler = tbl.isNonNative() && tbl.getStorageHandler().canSetColStatistics(tbl); if (!useStorageHandler && tbl.getSnapshotRef() != null) { - // the metastore holds only the table's statistics: a branch write has nowhere to store its own + // the metastore holds only the table's statistics; a branch has nowhere to store its own return 0; } // a storage handler persists all the statistics in a single batch @@ -238,7 +238,8 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce start = System. currentTimeMillis(); if (useStorageHandler) { boolean success = tbl.getStorageHandler().setColStatistics(tbl, colStats); - if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary())) { + // COLUMN_STATS_ACCURATE describes the table, so a branch write leaves it alone + if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary()) && tbl.getSnapshotRef() == null) { setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index d3e489f4a7c9..75e79ffce0ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -2025,8 +2026,8 @@ public static Range combineRange(Range range1, Range range2) { } public static boolean isPartitionStats(Table table, HiveConf conf) { - return conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && table.isPartitioned() - && (!table.isNonNative() || table.getStorageHandler().canSetColStatistics(table)); + return table.isPartitioned() && conf.getBoolVar(DDLUtils.isIcebergTable(table) ? + ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL : ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS); } public static boolean checkCanProvideStats(Table table) { From 4b4e1d5a90a56e387ab3e6fee03de12dfd292a36 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sat, 22 Aug 2026 19:21:49 +0300 Subject: [PATCH 05/15] HIVE-29829: Key written column statistics by the name a read looks them up under A write derived partition names its own way, so the statistics it stored were keyed differently from the way a read asks for them, and a read found nothing for a partition a write had just described. Both sides now take the name from the Iceberg spec. A write reaching some partitions, and an ANALYZE naming them, stand in for those partitions alone rather than for the whole table. A CTAS gathers partition-level statistics for the table it creates. --- .../mr/hive/HiveIcebergStorageHandler.java | 45 +++- .../iceberg/mr/hive/IcebergTableUtil.java | 46 +++- .../mr/hive/stats/ColStatsWritePolicy.java | 99 +++++++-- .../mr/hive/TestHiveIcebergStatistics.java | 204 ++++++++++++++++-- .../hive/stats/TestColStatsWritePolicy.java | 197 +++++++++++++++++ .../table/info/desc/DescTableOperation.java | 5 +- .../show/ShowPartitionsOperation.java | 3 + .../hive/ql/metadata/DummyPartition.java | 4 + .../hive/ql/metadata/HiveStorageHandler.java | 15 ++ .../parse/ColumnStatsAutoGatherContext.java | 21 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 89 +++++--- .../hive/ql/parse/SemanticAnalyzer.java | 8 + .../hadoop/hive/ql/parse/TransformSpec.java | 36 ++++ .../hive/ql/stats/ColStatsProcessor.java | 15 +- .../hadoop/hive/ql/stats/StatsUtils.java | 23 +- 15 files changed, 727 insertions(+), 83 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 8f1857f7f91e..12161d17cc2f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -37,6 +37,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -870,15 +871,42 @@ private String getStatsSource() { /** - * Extends a table-level increment with the previous snapshot's statistics. Returns whether the - * write may proceed: without previous statistics to extend, the increment is dropped - only a - * complete rewrite can restart the chain. + * Completes the statistics a write computed with the ones already stored, for the two writes that + * describe less than the whole table: an INSERT, whose table-level statistics cover the rows it + * added, and an INSERT OVERWRITE or ANALYZE of some partitions, which leaves the rest untouched. + *

+ * Which stored statistics may be drawn on differs with that. An INSERT sums its rows into them, so + * they have to account for every other row the table holds; statistics an unrecorded write has + * already outlived would lose that write for good, so the increment is dropped and only a complete + * rewrite can restart the chain. Partitions are copied rather than summed, so the untouched ones + * may come from the nearest ancestor, which is what a read of them returns anyway. + * + * @return whether the write may proceed */ private boolean checkAndMergeColStats(List statsNew, Table tbl, Snapshot snapshot) throws InvalidObjectException, IOException { - StatisticsFile statsOldSrc = IcebergTableUtil.getColStatsFile(tbl, snapshot.parentId(), conf); + boolean isTblLevel = statsNew.getFirst().getStatsDesc().isIsTblLevel(); + Long parentId = snapshot.parentId(); + // A write commits a snapshot of its own, so what it completes sits on the one before it. An + // ANALYZE commits none: it writes to the snapshot it read, where the statistics already are. + StatisticsFile statsOldSrc = isTblLevel ? + (parentId == null ? null : IcebergTableUtil.getColStatsFile(tbl, parentId, conf)) : + IcebergTableUtil.findColStatsFile(tbl, snapshot.snapshotId(), conf); if (statsOldSrc == null) { - return false; + // a partition describes itself, so with nothing stored there is nothing to carry and what + // was computed stands on its own; a table-level increment has nothing to add itself to + return !isTblLevel; + } + if (!isTblLevel) { + // The file holds one blob per partition. This write replaced the rows of the partitions it + // reached, so its statistics stand for those, and the partitions it never saw carry over. + List statsOld = IcebergTableUtil.readColStatsOrThrow(tbl, statsOldSrc, null); + // two blobs of one partition would leave nothing to merge into, so the later one answers + Map storedByPart = statsOld.stream().collect( + Collectors.toMap(stats -> stats.getStatsDesc().getPartName(), Function.identity(), (a, b) -> b)); + statsNew.forEach(stats -> storedByPart.remove(stats.getStatsDesc().getPartName())); + statsNew.addAll(storedByPart.values()); + return true; } List statsOld = IcebergTableUtil.readColStatsOrThrow(tbl, statsOldSrc, null); ColumnStatistics statsObjOld = new ColumnStatistics(null, statsOld); @@ -934,6 +962,13 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m } + @Override + public Function partitionNameResolver( + org.apache.hadoop.hive.ql.metadata.Table hmsTable, StructObjectInspector inspector) { + return IcebergTableUtil.partitionNameFunction( + IcebergTableUtil.getTable(conf, hmsTable.getTTable()), inspector); + } + private List getWriteSortTransformSpecs(Table table) { return table.sortOrder().fields().stream() .map(s -> IcebergTableUtil.getTransformSpec(table, s.transform().toString(), s.sourceId())) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 2c7ef7fe1938..cb102cc8ee2f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -75,6 +75,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.util.Sets; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -92,6 +93,7 @@ import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatistics; import org.apache.iceberg.PartitionStatisticsFile; @@ -109,6 +111,9 @@ import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdatePartitionSpec; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.InternalRecordWrapper; +import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -117,12 +122,14 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.mr.hive.serde.objectinspector.IcebergObjectInspector; import org.apache.iceberg.puffin.Blob; import org.apache.iceberg.puffin.BlobMetadata; import org.apache.iceberg.puffin.Puffin; import org.apache.iceberg.puffin.PuffinCompressionCodec; import org.apache.iceberg.puffin.PuffinReader; import org.apache.iceberg.puffin.PuffinWriter; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; @@ -354,9 +361,7 @@ static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotI * @return iceberg partition spec, always non-null */ public static PartitionSpec spec(Configuration configuration, Schema schema) { - List partitionBy = SessionStateUtil - .getResource(configuration, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) - .map(o -> (List) o).orElse(null); + List partitionBy = TransformSpec.fromQueryState(configuration); if (partitionBy == null) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); @@ -572,6 +577,41 @@ public static String toPartitionName(PartitionSpec spec, StructLike data) { return path.isEmpty() ? DummyPartition.VOID : path; } + /** + * Returns a function naming the partition a row belongs to, where the row holds no more than the + * columns the spec partitions on. The spec derives the tuple itself, so a name matches the one a + * write gives the files it puts in that partition. Deriving one costs the same as writing a row, + * so the function holds on to what that takes and is worth reusing over rows an inspector reads. + */ + public static Function partitionNameFunction(Table table, StructObjectInspector inspector) { + // the deserializer reads the row by position, so the schema has to follow the inspector rather + // than the table, and a name reaches here lowercased however the table spells it + Schema sourceSchema = new Schema(inspector.getAllStructFieldRefs().stream() + .map(field -> Preconditions.checkNotNull(table.schema().caseInsensitiveFindField(field.getFieldName()), + "Table %s partitions on %s, which it has no column for", table.name(), field.getFieldName())) + .toList()); + Deserializer deserializer = new Deserializer.Builder() + .schema(sourceSchema) + .sourceInspector(inspector) + .writerInspector((StructObjectInspector) IcebergObjectInspector.create(sourceSchema)) + .build(); + + // the whole schema, not the row's columns: PartitionKey needs an accessor for every field of + // the spec, including a void one, whose source column the row does not carry + GenericRecord record = GenericRecord.create(table.schema()); + InternalRecordWrapper wrapper = new InternalRecordWrapper(table.schema().asStruct()); + PartitionKey partitionKey = new PartitionKey(table.spec(), table.schema()); + + return row -> { + // a row whose every value is null is indistinguishable from no row at all once serialized + Record source = row == null ? null : deserializer.deserialize(row); + sourceSchema.columns().forEach(column -> + record.setField(column.name(), source == null ? null : source.getField(column.name()))); + partitionKey.partition(wrapper.wrap(record)); + return toPartitionName(table.spec(), partitionKey); + }; + } + /** * Builds a filter expression for data table operations (deleteFromRowFilter, FindFiles). * Only supports identity transforms. Keys are partition field names. diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java index f318c46dce9b..2d2992544397 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java @@ -20,11 +20,13 @@ package org.apache.iceberg.mr.hive.stats; import java.util.List; +import java.util.function.BooleanSupplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.ql.Context.RewritePolicy; +import org.apache.hadoop.hive.ql.parse.ColumnStatsSemanticAnalyzer; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; @@ -41,11 +43,41 @@ public enum ColStatsWritePolicy { /** Write the computed statistics, discarding the stored ones. */ REPLACE, - /** Write the computed statistics merged with the stored ones. */ + /** + * Write the computed statistics completed by the stored ones: a table-level write adds its rows + * to them, a partition-level one stands in for the partitions it reached. + */ MERGE, /** Write nothing: leave the stored statistics as they are. */ SKIP; + /** + * Everything the decision reads, so that it can be made without a session. {@code statsAccurate} + * is a supplier because answering it walks the table's metadata, and most cases never ask. + */ + record ColStatsWrite( + // what this write computed + boolean tableWideStats, + // the table it computed them for, and what it already holds for this snapshot + boolean keepsStatsPerPartition, + boolean partitioned, + BooleanSupplier statsAccurate, + // the statement that computed them + boolean analyze, + boolean analyzePartition, + boolean compaction, + boolean fullTableMajorCompaction, + // what its commit did to the rows + boolean holdsOnlyAddedRows, + boolean emptySnapshot, + boolean wroteNoRows, + boolean replacePartitions) { + + boolean isStatsAccurate() { + return statsAccurate.getAsBoolean(); + } + } + /** * What to do with the statistics a write computed. * @@ -55,43 +87,71 @@ public enum ColStatsWritePolicy { public static ColStatsWritePolicy resolve( Table tbl, Snapshot snapshot, List colStats, Configuration conf) { - if (!colStats.getFirst().getStatsDesc().isIsTblLevel()) { - // Only ANALYZE gathers stats for every partition. - return isAnalyzeOperation(conf) ? REPLACE : SKIP; + return resolve(new ColStatsWrite( + colStats.getFirst().getStatsDesc().isIsTblLevel(), + IcebergTableUtil.isPartitionStats(tbl, conf), + tbl.spec().isPartitioned(), + () -> IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf), + isAnalyze(conf), + isAnalyzePartition(conf), + SessionState.get() != null && SessionState.get().isCompaction(), + isFullTableMajorCompaction(conf), + holdsOnlyAddedRows(snapshot), + IcebergTableUtil.isEmptySnapshot(snapshot), + wroteNoRows(conf), + Boolean.parseBoolean( + snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP)))); + } + + /** The decision itself, over the facts alone, so that every case of it can be stated as one. */ + static ColStatsWritePolicy resolve(ColStatsWrite write) { + if (!write.tableWideStats()) { + return resolveForPartitions(write); } - if (IcebergTableUtil.isPartitionStats(tbl, conf)) { - // This table's readers want partition-level stats, which only ANALYZE produces. + if (write.keepsStatsPerPartition()) { + // the table stores its statistics per partition; these describe it as a whole return SKIP; } - if (SessionState.get().isCompaction()) { + if (write.compaction()) { // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. - return isFullTableMajorCompaction(conf) && !IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf) ? - REPLACE : SKIP; + return write.fullTableMajorCompaction() && !write.isStatsAccurate() ? REPLACE : SKIP; } // ANALYZE reads the whole table, so it replaces. - return isAnalyzeOperation(conf) ? REPLACE : resolveForWrite(tbl, snapshot, conf); + return write.analyze() ? REPLACE : resolveForWrite(write); + } + + /** What a statement does to the file holding one blob per partition. */ + private static ColStatsWritePolicy resolveForPartitions(ColStatsWrite write) { + if (write.analyze()) { + // what it read is what it named, not what the snapshot it reads happens to hold + return write.analyzePartition() ? MERGE : REPLACE; + } + if (write.holdsOnlyAddedRows() || write.emptySnapshot()) { + return REPLACE; + } + return write.wroteNoRows() || write.isStatsAccurate() ? SKIP : MERGE; } /** What an INSERT, INSERT OVERWRITE or CTAS does to the stored statistics. */ - private static ColStatsWritePolicy resolveForWrite(Table tbl, Snapshot snapshot, Configuration conf) { - if (IcebergTableUtil.isEmptySnapshot(snapshot)) { + private static ColStatsWritePolicy resolveForWrite(ColStatsWrite write) { + if (write.emptySnapshot()) { // The table is now empty, so stats of the rows it held must go. Must precede the // wroteNoRows check, which an emptying write also matches. return REPLACE; } - if (wroteNoRows(conf) || IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf)) { + if (write.wroteNoRows() || write.isStatsAccurate()) { // Nothing to record: the statement wrote no rows, or this snapshot already has stats. return SKIP; } - if (holdsOnlyAddedRows(snapshot)) { + if (write.holdsOnlyAddedRows()) { // Every row came from this write: a CTAS, a whole-table INSERT OVERWRITE, or the first // INSERT after a TRUNCATE. return REPLACE; } - if (Boolean.parseBoolean(snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP))) { + if (write.replacePartitions()) { // Iceberg flags whole-table and partition overwrites alike, so the partition spec decides: // only an unpartitioned table had every row replaced. - return tbl.spec().isPartitioned() ? SKIP : REPLACE; + return write.partitioned() ? SKIP : REPLACE; } // An INSERT: its stats cover the rows it added, the stored ones cover the rest. return MERGE; @@ -114,12 +174,17 @@ private static boolean wroteNoRows(Configuration conf) { .orElse(false); } - private static boolean isAnalyzeOperation(Configuration conf) { + private static boolean isAnalyze(Configuration conf) { return SessionStateUtil.getQueryState(conf) .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) .orElse(false); } + /** Whether the ANALYZE named the partitions it is for, leaving the rest of the table alone. */ + private static boolean isAnalyzePartition(Configuration conf) { + return SessionStateUtil.getResource(conf, ColumnStatsSemanticAnalyzer.ANALYZE_PARTITION).isPresent(); + } + /** Whether the compaction read the whole table: a minor one sets a file size threshold. */ private static boolean isFullTableMajorCompaction(Configuration conf) { return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)) && diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 581c12c2f705..3b5a9fca8c79 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -380,8 +380,8 @@ public void testStatsWithInsertOverwrite() { @Test public void testStatsWithPartitionedInsertOverwrite() { - // a partition overwrite maintains no partition-level statistics: the pre-existing ones keep - // serving as approximations until ANALYZE recomputes + // a partition overwrite replaces the statistics of the partitions it wrote, and carries the + // ones it never reached across unchanged assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "orders_iow"); @@ -393,20 +393,175 @@ public void testStatsWithPartitionedInsertOverwrite() { Assert.assertTrue(colStatsAccurate(identifier)); shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (5, 'a')"); - Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); - Assert.assertFalse(colStatsAccurate(identifier)); - Assert.assertFalse(readCurrentColStats(identifier).isEmpty()); - - shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); Assert.assertTrue(colStatsAccurate(identifier)); + List colStats = readCurrentColStats(identifier); - // p=a recomputed: the overwritten rows no longer bound its range + // p=a was overwritten, so the rows it no longer holds stop bounding its range ColumnStatisticsObj idA = colStatsObj(colStats, "p=a", "id"); Assert.assertEquals(5L, idA.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(5L, idA.getStatsData().getLongStats().getHighValue()); + // p=b was never written, so its statistics came across from the previous file ColumnStatisticsObj idB = colStatsObj(colStats, "p=b", "id"); Assert.assertEquals(7L, idB.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); + + // and a recompute of the whole table agrees with what the overwrite left behind + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + colStats = readCurrentColStats(identifier); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testPartitionScopedAnalyzeWithoutStoredColStats() { + // there is nothing stored to carry, so the partition it named is all the file has to hold + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_scoped_first"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + // an insert gathers nothing while the table keeps its statistics per partition + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + Assert.assertFalse(hasColStatsForCurrentSnapshot(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a"), colStatsPartNames(identifier)); + } + + @Test + public void testPartitionScopedAnalyzeAfterWholeTableOverwrite() { + // the snapshot it reads was written by something that covered the table, which says nothing + // about what the ANALYZE itself was pointed at + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_after_iow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (1, 'a'), (7, 'b')"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + Assert.assertEquals(7L, colStatsObj(readCurrentColStats(identifier), "p=b", "id") + .getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testFullAnalyzeRetiresVanishedPartition() { + // recomputing the whole table is a reset: a partition it no longer finds keeps no statistics, + // where one that named partitions would have carried them over + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_retire"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a", "p=b"), colStatsPartNames(identifier)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE p = 'b'"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(List.of("p=a"), colStatsPartNames(identifier)); + } + + @Test + public void testStatsWithPartitionScopedAnalyze() { + // an ANALYZE naming a partition recomputes that one and leaves the others as they were + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_scoped"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (9, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(9L, colStatsObj(readCurrentColStats(identifier), "p=a", "id") + .getStatsData().getLongStats().getHighValue()); + + // take the row that bounded p=a away, so only statistics that replace rather than combine can + // report the range that is left + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 9"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(2L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + // p=b was not named, so its statistics came across untouched + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testStatsWithTransformPartition() { + // a transform names its partition by the value it produced, which reads nothing like the sort + // ordinal the transform hands back + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_day"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (day(ts)) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, timestamp'2024-01-01 10:00:00'), (7, timestamp'2024-06-15 12:00:00')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, + colStatsObj(colStats, "ts_day=2024-01-01", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, + colStatsObj(colStats, "ts_day=2024-06-15", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testStatsWithTimestampIdentityPartition() { + // an identity timestamp is one Hive and Iceberg spell differently + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_ts"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (ts) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, timestamp'2024-01-01 10:00:00'), (7, timestamp'2024-06-15 12:00:00')"); + + checkColStat(identifier.name(), "id", true); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + } + + @Test + public void testStatsWithPartitionFieldsOutOfSchemaOrder() { + // the spec orders its fields as it likes, which the row carrying their values has to follow + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_order"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, b string, c string) " + + "PARTITIONED BY SPEC (c, b) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " VALUES (1, 'bb', 'cc'), (7, 'bb2', 'cc2')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, colStatsObj(colStats, "c=cc/b=bb", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "c=cc2/b=bb2", "id").getStatsData().getLongStats().getLowValue()); + } + + @Test + public void testStatsWithNullPartitionValue() { + // the rows a partition holds none of a value for still form a partition of their own + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_null"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, ts timestamp) " + + "PARTITIONED BY SPEC (day(ts)) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + + " VALUES (1, null), (7, timestamp'2024-06-15 12:00:00')"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(1L, colStatsObj(colStats, "ts_day=null", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(7L, + colStatsObj(colStats, "ts_day=2024-06-15", "id").getStatsData().getLongStats().getLowValue()); } @Test @@ -467,11 +622,6 @@ public void testStatsWithPartitionedCTAS() { testTables.propertiesForCreateTableSQL( ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())))); - if (statsSource.equals("iceberg")) { - // TODO: Propagate partition spec from CREATE statement to the ColumnStatsSemanticAnalyzer - shell.executeStatement("ANALYZE TABLE target COMPUTE STATISTICS FOR COLUMNS"); - } - checkColStat("target", "id", true); checkColStat("target", "dept", true); checkColStatMinMaxValue("target", "id", 0, 2); @@ -750,20 +900,42 @@ public void testAnalyzePartitionSpecRejected() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); createPartitionedCustomers(identifier); - // statistics are maintained for all partitions as a whole: a partition-scoped ANALYZE is - // rejected for non-native tables, for basic and column statistics alike + // the metastore holds one row count for the table, with nowhere to record a single partition's AssertHelpers.assertThrows( "Should reject partition-scoped basic-stats ANALYZE for non-native partitioned tables", IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS") ); + // and column statistics of one partition need a table that keeps them per partition + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); AssertHelpers.assertThrows( - "Should reject partition-scoped column-stats ANALYZE for non-native partitioned tables", + "Should reject partition-scoped column-stats ANALYZE when statistics are kept for the table", IllegalArgumentException.class, ErrorMsg.ANALYZE_PARTITION_NON_NATIVE.getMsg(), () -> shell.executeStatement( "ANALYZE TABLE " + identifier + " PARTITION (last_name='Brown') COMPUTE STATISTICS FOR COLUMNS") ); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, true); + } + + @Test + public void testStatsWithPartitionScopedInsertOverwrite() { + // naming the partition to overwrite reaches the same statistics as letting the rows choose it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_static"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + shell.executeStatement("INSERT OVERWRITE TABLE " + identifier + " PARTITION (p='a') VALUES (5)"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getHighValue()); } @Test diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java new file mode 100644 index 000000000000..f118951b6ddc --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.ColStatsWrite; +import org.junit.Test; + +import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.MERGE; +import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.REPLACE; +import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.SKIP; +import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.resolve; +import static org.junit.Assert.assertEquals; + +/** Every case of the decision, stated as one, without a session to make it in. */ +public class TestColStatsWritePolicy { + + @Test + public void analyzeOfTheWholeTableReplacesEveryPartition() { + assertEquals(REPLACE, resolve(partitionLevel().analyze().build())); + } + + @Test + public void analyzeOfNamedPartitionsKeepsTheOthers() { + assertEquals(MERGE, resolve(partitionLevel().analyzePartition().build())); + } + + @Test + public void analyzeIgnoresWhatTheSnapshotItReadsHolds() { + // an ANALYZE inherits the last write's snapshot; what that covered says nothing about its own + assertEquals(REPLACE, resolve(partitionLevel().analyze().holdsOnlyAddedRows().build())); + assertEquals(MERGE, resolve(partitionLevel().analyzePartition().holdsOnlyAddedRows().build())); + } + + @Test + public void analyzeRecomputesEvenWhenTheStoredStatisticsAreCurrent() { + assertEquals(REPLACE, resolve(partitionLevel().analyze().statsAccurate().wroteNoRows().build())); + } + + @Test + public void aWriteThatProducedEveryRowReplaces() { + assertEquals(REPLACE, resolve(partitionLevel().holdsOnlyAddedRows().build())); + assertEquals(REPLACE, resolve(partitionLevel().emptySnapshot().build())); + } + + @Test + public void aWriteThatReachedSomePartitionsKeepsTheOthers() { + assertEquals(MERGE, resolve(partitionLevel().build())); + } + + @Test + public void aWriteWithNothingToRecordLeavesThePartitionStatisticsAlone() { + assertEquals(SKIP, resolve(partitionLevel().wroteNoRows().build())); + assertEquals(SKIP, resolve(partitionLevel().statsAccurate().build())); + } + + @Test + public void tableWideStatisticsOfATableThatKeepsThemPerPartitionGoNowhere() { + assertEquals(SKIP, resolve(tableWideStats().keepsStatsPerPartition().analyze().build())); + } + + @Test + public void onlyAWholeTableCompactionRefreshesStaleStatistics() { + assertEquals(REPLACE, resolve(tableWideStats().fullTableMajorCompaction().build())); + assertEquals(SKIP, resolve(tableWideStats().compaction().build())); + assertEquals(SKIP, resolve(tableWideStats().fullTableMajorCompaction().statsAccurate().build())); + } + + @Test + public void tableWideAnalyzeReplaces() { + assertEquals(REPLACE, resolve(tableWideStats().analyze().build())); + } + + @Test + public void anEmptiedTableLosesTheStatisticsOfTheRowsItHeld() { + assertEquals(REPLACE, resolve(tableWideStats().emptySnapshot().wroteNoRows().build())); + } + + @Test + public void anInsertAddsItsRowsToTheStoredStatistics() { + assertEquals(MERGE, resolve(tableWideStats().build())); + } + + @Test + public void anOverwriteOfSomePartitionsCannotDescribeTheTable() { + assertEquals(SKIP, resolve(tableWideStats().replacePartitions().partitioned().build())); + assertEquals(REPLACE, resolve(tableWideStats().replacePartitions().build())); + } + + /** A write of per-partition statistics that neither covered the table nor left it empty. */ + private static Builder partitionLevel() { + return new Builder(); + } + + /** A write of statistics describing the table as a whole. */ + private static Builder tableWideStats() { + return new Builder().tableWideStats(); + } + + /** Names each fact, so a case reads as the statement it stands for. */ + private static final class Builder { + private boolean tableWideStats; + private boolean keepsStatsPerPartition; + private boolean analyze; + private boolean analyzePartition; + private boolean compaction; + private boolean fullTableMajorCompaction; + private boolean holdsOnlyAddedRows; + private boolean emptySnapshot; + private boolean wroteNoRows; + private boolean replacePartitions; + private boolean partitioned; + private boolean statsAccurate; + + private Builder tableWideStats() { + tableWideStats = true; + return this; + } + + private Builder keepsStatsPerPartition() { + keepsStatsPerPartition = true; + return this; + } + + private Builder analyze() { + analyze = true; + return this; + } + + private Builder analyzePartition() { + analyzePartition = true; + return analyze(); + } + + private Builder compaction() { + compaction = true; + return this; + } + + private Builder fullTableMajorCompaction() { + fullTableMajorCompaction = true; + return compaction(); + } + + private Builder holdsOnlyAddedRows() { + holdsOnlyAddedRows = true; + return this; + } + + private Builder emptySnapshot() { + emptySnapshot = true; + return this; + } + + private Builder wroteNoRows() { + wroteNoRows = true; + return this; + } + + private Builder replacePartitions() { + replacePartitions = true; + return this; + } + + private Builder partitioned() { + partitioned = true; + return this; + } + + private Builder statsAccurate() { + statsAccurate = true; + return this; + } + + private ColStatsWrite build() { + return new ColStatsWrite(tableWideStats, keepsStatsPerPartition, partitioned, () -> statsAccurate, + analyze, analyzePartition, compaction, fullTableMajorCompaction, holdsOnlyAddedRows, + emptySnapshot, wroteNoRows, replacePartitions); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index da691dcaf79c..8f91d27b4361 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -153,7 +154,9 @@ private void getColumnsNoColumnPath(Table table, Partition partition, List valueMap.put(k, Longs.tryParse(v))); - numParts = sh.getPartitionNames(table).size(); + numParts = (int) sh.getPartitionNames(table).stream() + .filter(name -> !DummyPartition.isVoid(name)) + .count(); } else { PartitionIterable partitions = new PartitionIterable(context.getDb(), table, null, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java index 58a829e3de22..60fae4c9435b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionsOperation.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLOperation; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.ddl.ShowUtils; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -67,6 +68,8 @@ public int execute() throws HiveException { parts = context.getDb().getPartitionNames(tbl, desc.getLimit()); } + parts = parts.stream().filter(name -> !DummyPartition.isVoid(name)).toList(); + // write the results in the file try (DataOutputStream outStream = ShowUtils.getOutputStream(new Path(desc.getResFile()), context)) { ShowPartitionsFormatter formatter = ShowPartitionsFormatter.getFormatter(context.getConf()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 042c4e2b5566..1df3300bfdcf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -43,6 +43,10 @@ public class DummyPartition extends Partition { */ public static final String VOID = "__NO_PARTITION__"; + /** + * Whether the name is the synthetic partition's. No statement can name that partition, so + * listings shown to users skip it, while planning and statistics keep counting its rows. + */ public static boolean isVoid(String partName) { return VOID.equals(partName); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 6ff3742833bc..379f89dc3d4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils; import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCommitter; @@ -81,6 +82,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Function; /** * HiveStorageHandler defines a pluggable interface for adding @@ -604,6 +606,19 @@ default List getPartitionTransformSpec(org.apache.hadoop.hive.ql. return null; } + /** + * Returns a function naming the partition a row belongs to. Statistics join on this name, so a + * handler must derive it the way it derives the names of the partitions it writes. Deriving one + * may cost as much as writing a row, so the function is asked for once for all the rows an + * inspector reads, and is neither reentrant nor thread-safe. + * @param table the HMS table, must be non-null + * @param inspector the inspector of a row holding the columns the partitioning is derived from + */ + default Function partitionNameResolver( + org.apache.hadoop.hive.ql.metadata.Table table, StructObjectInspector inspector) { + throw new UnsupportedOperationException(getClass().getName() + " does not name partitions"); + } + /** * Creates a DynamicPartitionCtx instance that will be set up by the storage handler itself. Useful for non-native diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 5adff4b08655..bf851fa8e5cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -70,6 +70,8 @@ public class ColumnStatsAutoGatherContext { private Table tbl; private Map partSpec; private Context origCtx; + + private boolean isTableLevel = true; public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, Operator op, Table tbl, Map partSpec, @@ -123,14 +125,22 @@ public void insertAnalyzePipeline() throws SemanticException { */ public void insertTableValuesAnalyzePipeline() throws SemanticException { // Instead of starting from analyze statement, we just generate the Select plan - boolean isPartitionStats = StatsUtils.isPartitionStats(tbl, conf) && !tbl.hasNonNativePartitionSupport(); - if (isPartitionStats) { + // the table is not created yet, so only the CREATE statement says how it partitions its rows + List partTransformSpec = tbl.hasNonNativePartitionSupport() ? + TransformSpec.fromQueryState(conf) : null; + + boolean isPartitionStats = partTransformSpec != null ? + StatsUtils.isPartitionStatsEnabled(tbl, conf) : StatsUtils.isPartitionStats(tbl, conf); + + if (isPartitionStats && partTransformSpec == null) { partSpec = new HashMap<>(); List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); partKeys.forEach(k -> partSpec.put(k, null)); - } - String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, conf, partSpec, isPartitionStats); + isTableLevel = !isPartitionStats; + + String command = ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, conf, + partTransformSpec, partSpec, isPartitionStats); insertAnalyzePipeline(command, true); } @@ -172,6 +182,9 @@ private Operator genSelOp(String command, boolean rewritten, Context origCtx) if (rewritten) { // Create the context object that is needed to store the column stats this.analyzeRewrite = ColumnStatsSemanticAnalyzer.genAnalyzeRewriteContext(conf, tbl); + // a table the CREATE has yet to make cannot be asked how it partitions its rows, so the + // scope the statement was written for is the one that holds + this.analyzeRewrite.setTblLvl(isTableLevel); // The analyze statement has already been rewritten, we just need to create the AST // and the corresponding semantic analyzer diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 601c346125e2..2a785d4af081 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsField; import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsType; @@ -64,6 +65,12 @@ * */ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer { + /** + * Set while an ANALYZE that named partitions is compiled: its statistics stand for those alone, + * so what the table keeps for the rest has to survive them. + */ + public static final String ANALYZE_PARTITION = "hive.stats.analyze.partition"; + private static final Logger LOG = LoggerFactory .getLogger(ColumnStatsSemanticAnalyzer.class); private static final LogHelper CONSOLE = new LogHelper(LOG); @@ -145,8 +152,8 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } try { // for static partition, it may not exist when HIVE_STATS_COL_AUTOGATHER is - // set to true - if (context == null && partValsSpecified > 0) { + // set to true. A table partitioned outside the metastore keeps no partition there to find. + if (context == null && partValsSpecified > 0 && !tbl.hasNonNativePartitionSupport()) { if ((partValsSpecified == tbl.getPartitionKeys().size()) && (db.getPartition(tbl, partSpec, false, null, false) == null)) { throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() @@ -159,7 +166,7 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } // User might have only specified partial list of partition keys, in which case add other partition keys in partSpec - List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); + List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartCols()); for (String partKey : partKeys) { if (!partSpec.containsKey(partKey)) { partSpec.put(partKey, null); @@ -174,16 +181,24 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } } - private static CharSequence genPartitionClause(Table tbl, Map partSpec, HiveConf conf) { - boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); - - StringBuilder whereClause = new StringBuilder(" where ").append( + /** The predicate naming the partitions an ANALYZE was pointed at, empty when it named none. */ + private static CharSequence genPartitionPredicate(Table tbl, Map partSpec, HiveConf conf) { + if (partSpec == null || partSpec.values().stream().noneMatch(Objects::nonNull)) { + return ""; + } + return new StringBuilder(" where ").append( partSpec.entrySet().stream() .filter(part -> part.getValue() != null) - .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + genPartValueString(getColTypeOf(tbl, part.getKey()), part.getValue())) .collect(Collectors.joining(" and ")) ); + } + + private static CharSequence genPartitionClause(Table tbl, Map partSpec, HiveConf conf) { + boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); + + StringBuilder whereClause = new StringBuilder(genPartitionPredicate(tbl, partSpec, conf)); StringBuilder groupByClause = new StringBuilder(" group by ").append( tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) @@ -247,8 +262,9 @@ protected static List getFieldSchemasByColName(Table tbl, List partSpec, boolean isPartitionStats) { - String rewritten = genRewrittenQuery(tbl, columnSchemas, conf, partSpec, isPartitionStats, false); + List partTransformSpec, Map partSpec, boolean isPartitionStats) { + String rewritten = + genRewrittenQuery(tbl, columnSchemas, conf, partTransformSpec, partSpec, isPartitionStats, false); isRewritten = true; return rewritten; } @@ -257,14 +273,15 @@ private String genRewrittenQuery(FieldSchemas columnSchemas, HiveConf conf, * Generates a SQL statement that will compute the stats for all columns * included in the input table. */ - protected static String genRewrittenQuery(Table tbl, - HiveConf conf, Map partSpec, boolean isPartitionStats) { + protected static String genRewrittenQuery(Table tbl, HiveConf conf, + List partTransformSpec, Map partSpec, boolean isPartitionStats) { return ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, getStatsEligibleFieldSchemas(tbl), conf, - partSpec, isPartitionStats, true); + partTransformSpec, partSpec, isPartitionStats, true); } - private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, - HiveConf conf, Map partSpec, boolean isPartitionStats, boolean useTableValues) { + private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, HiveConf conf, + List partTransformSpec, Map partSpec, boolean isPartitionStats, + boolean useTableValues) { StringBuilder rewrittenQueryBuilder = new StringBuilder("select "); StringBuilder columnNamesBuilder = new StringBuilder(); @@ -293,7 +310,11 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, } if (isPartitionStats) { - if (tbl.hasNonNativePartitionSupport()) { + if (partTransformSpec != null) { + // a write produces one spec, so its rows can be grouped by the transforms themselves, + // and carry the values those were applied to for the table to name the group + rewrittenQueryBuilder.append(", ").append(TransformSpec.toSourceStruct(partTransformSpec, conf)); + } else if (tbl.hasNonNativePartitionSupport()) { // group every row by its read-side partition name rewrittenQueryBuilder.append(", ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); } else { @@ -333,10 +354,16 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, // If partition level statistics is requested, add predicate and group by as needed to rewritten // query if (isPartitionStats) { - if (!tbl.hasNonNativePartitionSupport()) { + if (partTransformSpec != null) { + rewrittenQueryBuilder.append("\ngroup by ").append(partTransformSpec.stream() + .map(spec -> spec.toHiveExpr(conf)).collect(Collectors.joining(", "))); + } else if (!tbl.hasNonNativePartitionSupport()) { rewrittenQueryBuilder.append(genPartitionClause(tbl, partSpec, conf)); } else { - rewrittenQueryBuilder.append("\ngroup by ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + // the name groups rows of every spec alike; the predicate keeps the scan to the partitions + // the statement named, which only the current spec's own columns can express + rewrittenQueryBuilder.append(genPartitionPredicate(tbl, partSpec, conf)) + .append("\ngroup by ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); } } @@ -618,11 +645,10 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { originalTree = ast; boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) || StatsUtils.isPartitionStats(tbl, conf); - // a partition-scoped column statistics ANALYZE cannot be honored for non-native tables: the stats - // of all partitions are rewritten as a whole, so it would drop every other partition's statistics - // (table-level ANALYZE of a partitioned table still computes all partitions - only the explicit - // partition spec is rejected; the auto-gather path merges instead and stays unaffected) - validateUnsupportedPartitionClause(tbl, AnalyzeCommandUtils.isPartitionLevelStats(ast)); + // naming partitions asks for the statistics of those alone, which a table keeping one set for + // itself has nowhere to put + validateUnsupportedPartitionClause(tbl, + AnalyzeCommandUtils.isPartitionLevelStats(ast) && !StatsUtils.isPartitionStats(tbl, conf)); Map partSpec = (isPartitionStats) ? AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf) : null; @@ -634,7 +660,10 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partSpec, isPartitionStats); + if (partSpec != null && partSpec.values().stream().anyMatch(Objects::nonNull)) { + SessionStateUtil.addResourceOrThrow(conf, ANALYZE_PARTITION, Boolean.TRUE); + } + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, null, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); } else { // Not an analyze table column compute statistics statement - don't do any rewrites @@ -682,9 +711,9 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) tbl = AnalyzeCommandUtils.getTable(ast, this); - boolean isPartitionStats = (AnalyzeCommandUtils.isPartitionLevelStats(ast) - || StatsUtils.isPartitionStats(tbl, conf)) && !tbl.hasNonNativePartitionSupport(); - + boolean isPartitionStats = AnalyzeCommandUtils.isPartitionLevelStats(ast) + || StatsUtils.isPartitionStats(tbl, conf); + List partTransformSpec = null; Map partSpec = null; List columnSchemas = getColumnsFromAst(ast); @@ -692,11 +721,15 @@ public ASTNode rewriteAST(ASTNode ast, ColumnStatsAutoGatherContext context) if (isPartitionStats) { partSpec = AnalyzeCommandUtils.getPartKeyValuePairsFromAST(tbl, ast, conf); handlePartialPartitionSpec(partSpec, context); + if (tbl.hasNonNativePartitionSupport()) { + // a write produces one spec, so the transforms group its rows + partTransformSpec = tbl.getStorageHandler().getPartitionTransformSpec(tbl); + } } rewrittenColumnSchemas = new FieldSchemas(columnSchemas); isTableLevel = !isPartitionStats; - rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partSpec, isPartitionStats); + rewrittenQuery = genRewrittenQuery(rewrittenColumnSchemas, conf, partTransformSpec, partSpec, isPartitionStats); rewrittenTree = genRewrittenTree(rewrittenQuery); return rewrittenTree; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 6bcbd346b235..c370110a68cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -266,6 +266,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; @@ -8848,6 +8849,13 @@ private void createPreInsertDesc(Table table, boolean overwrite) { private void genAutoColumnStatsGatheringPipeline(Table table, Map partSpec, Operator curr, boolean isInsertInto, boolean useTableValueConstructor) throws SemanticException { + if (isInsertInto && table.hasNonNativePartitionSupport() && StatsUtils.isPartitionStats(table, conf)) { + // this table keeps its column statistics per partition, and an insert reaches too few of them + // to pay for grouping the gather by partition; they stand until something covers the table + LOG.debug("Skipping column stats autogather for insert into partition-level table {}", + table.getTableName()); + return; + } LOG.info("Generate an operator pipeline to autogather column stats for table " + table.getTableName() + " in query " + ctx.getCmd()); ColumnStatsAutoGatherContext columnStatsAutoGatherContext = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java index bea31ba62f9b..a205f4099c77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java @@ -19,8 +19,11 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import java.util.List; +import java.util.Set; import java.util.Locale; import java.util.Optional; import java.util.regex.Matcher; @@ -83,6 +86,39 @@ public String transformTypeString() { "[" + width + "]").orElse(""); } + /** + * The partition transforms the statement being compiled declared, or null if it declared none. + * A CREATE has to be read this way: the table it describes does not exist to be asked yet. + */ + @SuppressWarnings("unchecked") + public static List fromQueryState(Configuration conf) { + return SessionStateUtil.getResource(conf, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) + .map(spec -> (List) spec) + .orElse(null); + } + + + /** + * Builds the struct of source values a stats gather carries alongside each group. The transforms + * decide which rows form a partition, but only the values they were applied to let the table name + * it, and every row of a group belongs to one partition, so any of them answers for the group. + */ + public static String toSourceStruct(List partTransformSpec, Configuration conf) { + Set groupedColumns = partTransformSpec.stream() + .filter(spec -> spec.getTransformType() == TransformType.IDENTITY) + .map(TransformSpec::getColumnName) + .collect(Collectors.toSet()); + + return partTransformSpec.stream() + .map(TransformSpec::getColumnName).distinct() + .map(columnName -> { + String identifier = unparseIdentifier(columnName, conf); + // an identity transform groups by the column itself, which already answers for the group + return "'" + columnName + "', " + + (groupedColumns.contains(columnName) ? identifier : "min(" + identifier + ")"); + }) + .collect(Collectors.joining(", ", "named_struct(", ")")); + } public String toHiveExpr(Configuration conf) { String identifier = unparseIdentifier(columnName, conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 15453a6ec385..9adb12bbc5c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -100,6 +101,9 @@ public int process(Hive db, Table tbl) throws Exception { private boolean constructColumnStatsFromPackedRows(Table tbl, List stats, long maxNumStats) throws HiveException, MetaException, IOException { String partName = null; + // the rows of one fetch share an inspector, so the table is asked once how to name them + Function partitionNameResolver = null; + List colName = colStatDesc.getColName(); List colType = colStatDesc.getColType(); boolean isTblLevel = colStatDesc.isTblLevel(); @@ -142,8 +146,15 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List= 1 && !colStats.isEmpty()) { - stats.setDataSize(betterDS); - } + stats.setDataSize((betterDS < 1 || colStats.isEmpty()) ? ds : betterDS); } // infer if any column can be primary key based on column statistics inferAndSetPrimaryKey(stats.getNumRows(), colStats); @@ -470,9 +468,7 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p addPartitionColumnStats(conf, partitionColsToRetrieve, schema, partList, columnStats); long betterDS = getDataSizeFromColumnStats(nr, columnStats); - if (betterDS >= 1 && !columnStats.isEmpty()) { - stats.setDataSize(betterDS); - } + stats.setDataSize((betterDS < 1 || columnStats.isEmpty()) ? ds : betterDS); // infer if any column can be primary key based on column statistics inferAndSetPrimaryKey(stats.getNumRows(), columnStats); @@ -2026,7 +2022,20 @@ public static Range combineRange(Range range1, Range range2) { } public static boolean isPartitionStats(Table table, HiveConf conf) { - return table.isPartitioned() && conf.getBoolVar(DDLUtils.isIcebergTable(table) ? + return table.isPartitioned() && isPartitionStatsEnabled(table, conf); + } + + /** + * Whether this table's statistics are kept per partition, leaving aside whether it has any. A + * CREATE has to ask this way, since the table it is about to write does not exist to be asked. + */ + public static boolean isPartitionStatsEnabled(Table table, HiveConf conf) { + // the metastore keeps a single row of column statistics per table, with nowhere to put a + // partition's, so a table partitioned outside it can only keep them per partition itself + if (table.isNonNative() && !table.getStorageHandler().canSetColStatistics(table)) { + return false; + } + return conf.getBoolVar(DDLUtils.isIcebergTable(table) ? ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL : ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS); } From 7077640d294f118fb3f179c56ba03f5cc6d1a94d Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 24 Aug 2026 15:35:49 +0300 Subject: [PATCH 06/15] HIVE-29829: Judge stored column statistics by the snapshot their file was written for A statistics file holds only what was valid for the snapshot it was written for, so statsFile.snapshotId() is the only anchor a read needs. A merge carries an entry forward only when no write since changed its partition, and a read serves what nothing changed after the file; partsFound reports how much of the scan is covered, so a caller needing an exact answer refuses and one estimating reports PARTIAL. Ordering by position on the ancestry rather than by sequence number fixes format version 1 tables, which number every snapshot 0 and so judged every partition fresh no matter what had been written since. A compaction reads the settings its table carries for it, and did so only after the query it runs and the statistics it gathers had already been settled from the settings it did not have. They are in hand first now, so a table asking to keep statistics per partition is heard by the compaction of one of its partitions, and one that asks for nothing of the sort still has that compaction leave its statistics alone. --- .../org/apache/hadoop/hive/conf/HiveConf.java | 4 + .../mr/hive/HiveIcebergStorageHandler.java | 38 +- .../iceberg/mr/hive/IcebergTableUtil.java | 139 ++++- .../compaction/IcebergQueryCompactor.java | 14 + .../mr/hive/stats/ColStatsWritePolicy.java | 39 +- .../mr/hive/TestHiveIcebergStatistics.java | 289 ++++++++++ .../hive/stats/TestColStatsWritePolicy.java | 38 +- .../iceberg_colstats_analyze_evolution.q | 60 +++ .../positive/iceberg_colstats_staleness.q | 59 +++ .../positive/iceberg_colstats_write_paths.q | 51 ++ .../iceberg_compaction_colstats_compute.q | 55 ++ ..._iceberg_copy_on_write_unpartitioned.q.out | 420 +++++++-------- .../positive/delete_iceberg_mixed.q.out | 36 +- .../positive/dynamic_semijoin_reduction.q.out | 18 +- .../iceberg_colstats_analyze_evolution.q.out | 433 +++++++++++++++ .../iceberg_colstats_granularity.q.out | 6 +- .../positive/iceberg_colstats_staleness.q.out | 426 +++++++++++++++ .../iceberg_colstats_write_paths.q.out | 377 ++++++++++++++ .../iceberg_insert_overwrite_partition.q.out | 246 ++++----- ...nsert_overwrite_partition_transforms.q.out | 40 +- .../iceberg_compaction_colstats_compute.q.out | 342 ++++++++++++ ...ge_iceberg_copy_on_write_partitioned.q.out | 16 +- .../src/test/results/positive/row_count.q.out | 14 +- ..._iceberg_copy_on_write_unpartitioned.q.out | 492 +++++++++--------- .../update_iceberg_partitioned_avro.q.out | 4 +- .../update_iceberg_partitioned_orc.q.out | 4 +- .../update_iceberg_partitioned_parquet.q.out | 4 +- .../vectorized_iceberg_merge_mixed.q.out | 183 ++++--- .../positive/write_iceberg_branch.q.out | 68 +-- .../resources/testconfiguration.properties | 1 + .../hive/ql/metadata/HiveStorageHandler.java | 11 + .../hadoop/hive/ql/stats/BasicStatsTask.java | 11 +- .../hadoop/hive/ql/stats/StatsUtils.java | 12 +- 33 files changed, 3169 insertions(+), 781 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index ec1835e63f2b..755b4a43e077 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2137,6 +2137,10 @@ public static enum ConfVars { "compaction of one current-spec partition refreshes it. Plain INSERT maintains no\n" + "partition-level statistics: its partitions read as stale until recomputed. After changing\n" + "this, statistics of the other granularity are ignored until recomputed."), + HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK("hive.iceberg.stats.max.snapshot.lookback", 20, + "How many snapshots a read of per partition column statistics walks back through to tell\n" + + "which partitions the writes since have changed. Each one costs reading the manifests it\n" + + "wrote, so a file further back than this cannot be judged and is not served."), HIVE_ICEBERG_EXPIRE_SNAPSHOT_NUMTHREADS("hive.iceberg.expire.snapshot.numthreads", 4, "The number of threads to be used for deleting files during expire snapshot. If set to 0 or below it uses the" + " default DirectExecutorService"), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 12161d17cc2f..e7e384ca7389 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -710,6 +710,14 @@ private Table getTable(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { return IcebergTableUtil.getTable(conf, hmsTable.getTTable(), skipCache); } + @Override + public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, boolean partitionLevel) { + // asked of the configuration alone: a CREATE has to ask before the table it describes exists, + // and whether that table has partitions at all is settled by the caller + return canSetColStatistics(hmsTable) && + partitionLevel == HiveConf.getBoolVar(conf, ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL); + } + @Override public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { return HiveMetaHook.ICEBERG.equals(getStatsSource()); @@ -766,6 +774,11 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta return Lists.newArrayList(); } + // this returns the whole-table statistics, so judge the whole-table file: a write since it was + // written leaves them stale, and the metastore withholds its own on the same terms + if (!IcebergTableUtil.colStatsAccurate(table, snapshot, false)) { + return Lists.newArrayList(); + } Predicate filter; if (colNames != null) { Set columns = Sets.newHashSet(colNames); @@ -794,14 +807,27 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION); double ndvTuner = MetastoreConf.getDoubleVar(getConf(), MetastoreConf.ConfVars.STATS_NDV_TUNER); + // per partition blobs are what is read below, so judge the per partition file + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(table, snapshot.snapshotId(), true); + if (statsFile == null) { + return new AggrStats(Collections.emptyList(), 0); + } Set partitions = Sets.newHashSet(partNames); - Predicate filter = metadata -> - partitions.contains(metadata.properties().get(PARTITION)); + // a partition written since the file was written is no longer described by it + Predicate upToDate = + IcebergTableUtil.upToDateColStats(table, snapshot, statsFile, conf, true); + Predicate filter = metadata -> { + String partition = metadata.properties().get(PARTITION); + return partitions.contains(partition) && upToDate.test(partition); + }; List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), true, filter); + Set asked = Sets.newHashSet(colNames); partStats.forEach(colStats -> - colStats.getStatsObj().removeIf(statsObj -> !colNames.contains(statsObj.getColName()))); + colStats.getStatsObj().removeIf(statsObj -> !asked.contains(statsObj.getColName()))); + // the metastore counts a partition as found only when it has every column asked about + partStats.removeIf(colStats -> colStats.getStatsObj().size() != colNames.size()); List colStatsList = MetaStoreServerUtils.aggrPartitionStats(partStats, MetaStoreUtils.getDefaultCatalog(conf), hmsTable.getDbName(), hmsTable.getTableName(), @@ -905,6 +931,12 @@ private boolean checkAndMergeColStats(List statsNew, Table tbl Map storedByPart = statsOld.stream().collect( Collectors.toMap(stats -> stats.getStatsDesc().getPartName(), Function.identity(), (a, b) -> b)); statsNew.forEach(stats -> storedByPart.remove(stats.getStatsDesc().getPartName())); + // Carry what no write since has changed. This is the only place that can decide it without + // a reader walking the history itself, and an entry that no longer describes its partition + // is of no use to any read. + Predicate stillHolds = + IcebergTableUtil.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); + storedByPart.keySet().removeIf(part -> !stillHolds.test(part)); statsNew.addAll(storedByPart.values()); return true; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index cb102cc8ee2f..e69b15967a04 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -30,7 +30,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -81,6 +80,7 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DataOperations; import org.apache.iceberg.DeleteFiles; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.GenericBlobMetadata; @@ -101,6 +101,7 @@ import org.apache.iceberg.PartitionsTable; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.StatisticsFile; @@ -165,6 +166,7 @@ public class IcebergTableUtil { private static final String NULL_VALUE = "null"; static final String PARTITION_FIELD = "partition"; + private static final String MODIFIED_PARTITIONS_PREFIX = "modifiedPartitions."; static final String STATS = "/stats/snap-"; private IcebergTableUtil() { @@ -284,9 +286,18 @@ static Snapshot getTableSnapshot(Table table, String snapshotRef) { * table maintains, which is the one every write produces. */ static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { + return getColStatsFile(table, snapshotId, isPartitionStats(table, conf)); + } + + static StatisticsFile getColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + // A rewrite leaves every row in the partition it was already in, and so separates statistics + // from nothing - unless the table has evolved, where compaction selects the rows of the older + // specs and writes them under the current one. That is the same condition the compactor + // branches on, and only per partition statistics can tell the difference. + boolean rewritesKeepPartitions = !partitionLevel || table.specs().size() == 1; return colStatsFileOf( - table, snapshotId, isPartitionStats(table, conf), - snapshot -> !DataOperations.REPLACE.equals(snapshot.operation())); + table, snapshotId, partitionLevel, + snapshot -> !rewritesKeepPartitions || !DataOperations.REPLACE.equals(snapshot.operation())); } /** @@ -298,7 +309,7 @@ static StatisticsFile findColStatsFile(Table table, long snapshotId, Configurati return findColStatsFile(table, snapshotId, isPartitionStats(table, conf)); } - private static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { // a snapshot holding no rows (truncate) ends the walk: what precedes it didn't survive return colStatsFileOf( table, snapshotId, partitionLevel, IcebergTableUtil::isEmptySnapshot); @@ -834,14 +845,12 @@ private static List getPartitionNames(Configuration conf, .filter(evaluator::eval) .transform(row -> { PartitionSpec spec = icebergTable.specs().get(row.get(SPEC_IDX, Integer.class)); - if (!spec.isPartitioned()) { - return null; - } + // rows written before the table was partitioned: toPartitionName gives them the + // synthetic partition name PartitionData data = toPartitionData( row.get(PART_IDX, StructProjection.class), partitionType, spec.partitionType()); return Maps.immutableEntry(toPartitionName(spec, data), spec.specId()); }) - .filter(Objects::nonNull) .toSortedList(specIdComparator).stream() .map(Map.Entry::getKey) .toList(); @@ -920,10 +929,12 @@ public static TransformSpec getTransformSpec(Table table, String transformName, /** * Writes the column statistics as the table's statistics file for the current snapshot: one blob * per column at table level, one blob per partition otherwise, each carrying the partition name - * the read side joins on. + * the read side joins on. Everything written describes the snapshot it is written for, so a read + * asks only what happened after it. */ @SuppressWarnings("checkstyle:CyclomaticComplexity") - static boolean writeColStats(Table tbl, Snapshot snapshot, List colStats, Configuration conf) { + static boolean writeColStats(Table tbl, Snapshot snapshot, List colStats, + Configuration conf) { try { StatisticsFile statisticsFile; String statsPath = tbl.location() + STATS + UUID.randomUUID(); @@ -1016,13 +1027,119 @@ static boolean writeColStats(Table tbl, Snapshot snapshot, List partitionsChangedSince(Table table, Snapshot snapshot, long sinceSnapshotId, + Configuration conf, boolean capped) { + // the walk reads manifests, and every column of a query asks the same question of them + String cacheKey = MODIFIED_PARTITIONS_PREFIX + table.name() + '.' + snapshot.snapshotId() + '.' + + sinceSnapshotId + '.' + capped; + Optional cached = SessionStateUtil.getResource(conf, cacheKey); + if (cached.isPresent()) { + return ((Optional>) cached.get()).orElse(null); + } + Set changed = walkPartitionsChangedSince(table, snapshot, sinceSnapshotId, conf, capped); + SessionStateUtil.addResource(conf, cacheKey, Optional.ofNullable(changed)); + return changed; + } + + private static Set walkPartitionsChangedSince(Table table, Snapshot snapshot, + long sinceSnapshotId, Configuration conf, boolean capped) { + // the bound is what a read will wait for; a write settles its file for good, so it walks the + // whole way + int lookback = capped ? + HiveConf.getIntVar(conf, ConfVars.HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK) : Integer.MAX_VALUE; + Set changed = Sets.newHashSet(); + boolean rewritesKeepPartitions = table.specs().size() == 1; + Snapshot current = snapshot; + + for (int read = 0; current != null && current.snapshotId() != sinceSnapshotId; ) { + // A rewrite (REPLACE) leaves every row in the partition it was already in, unless the table + // has evolved: compaction then selects the rows of the older specs and writes them under the + // current one. An overwrite of partitions commits OVERWRITE, despite the name of its API. + if (!rewritesKeepPartitions || !DataOperations.REPLACE.equals(current.operation())) { + // only the snapshots whose manifests are read cost anything, so only they are counted + if (read == lookback) { + LOG.info("Stopped after {} snapshots of {}; the statistics written at snapshot {} " + + "cannot be judged", lookback, table.name(), sinceSnapshotId); + return null; + } + if (!collectChangedPartitions(table, current, changed)) { + return null; + } + read++; + } + Long parentId = current.parentId(); + current = parentId != null ? table.snapshot(parentId) : null; + } + // the file has to sit on the history walked, or what happened in between is unknown + return current == null ? null : changed; + } + + /** + * Records which partitions a snapshot changed, naming each file under its own spec. False when + * a delete of no partition is reached, since it applies to the rows of every one and names none. + * Stored + * entries are named the same way: an ANALYZE names each group after the file its rows came from, + * and a write only ever lands in a partition of the spec current when it ran. + */ + private static boolean collectChangedPartitions(Table table, Snapshot snapshot, Set changed) { + for (ContentFile file : changedFiles(table, snapshot)) { + PartitionSpec spec = table.specs().get(file.specId()); + // an equality delete of no partition applies to the rows of every one, and names none of them + if (file.content() != FileContent.DATA && !spec.isPartitioned() && table.spec().isPartitioned()) { + return false; + } + changed.add(toPartitionName(spec, file.partition())); + } + return true; + } + + /** Every file a snapshot added or removed, whether it holds rows or deletes them. */ + private static Iterable> changedFiles(Table table, Snapshot snapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(snapshot).build(); + return Iterables.concat( + changes.addedDataFiles(), changes.removedDataFiles(), + changes.addedDeleteFiles(), changes.removedDeleteFiles()); + } + + /** + * Whether a stored entry still describes its partition. A file describes the snapshot it was + * written for, so only what happened after it matters. False for all of them when the writes in + * between cannot be traced. + */ + static Predicate upToDateColStats(Table table, Snapshot snapshot, + StatisticsFile statsFile, Configuration conf, boolean capped) { + Set changed = + partitionsChangedSince(table, snapshot, statsFile.snapshotId(), conf, capped); + return partition -> changed != null && !changed.contains(partition); } /** The stored statistics describing the whole table. */ diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java index 387d3d983f76..f2514b70b1f1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.parse.TransformSpec; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext; +import org.apache.hadoop.hive.ql.txn.compactor.CompactorUtil; import org.apache.hadoop.hive.ql.txn.compactor.QueryCompactor; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -71,6 +72,10 @@ public boolean run(CompactorContext context) throws IOException, HiveException, HiveConf conf = new HiveConf(context.getConf()); CompactionInfo ci = context.getCompactionInfo(); + // the settings a table carries for its compactions decide what the query reads and what it + // measures, so they are in hand before either is settled + CompactorUtil.overrideConfProps(conf, ci, tblProperties); + String compactionQuery = buildCompactionQuery(context, compactTableName, conf); SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties); @@ -111,6 +116,15 @@ private String buildCompactionQuery(CompactorContext context, String compactTabl } } + // only a compaction that rewrote the whole table, or a whole partition of a table keeping + // statistics per partition, may store what it measured + boolean computeColStats = ci.isMajorCompaction() && (ci.partName == null ? + !icebergTable.spec().isPartitioned() : + IcebergTableUtil.isPartitionStats(icebergTable, conf)); + + boolean genericStats = HiveConf.getBoolVar(conf, ConfVars.HIVE_COMPACTOR_GATHER_STATS); + HiveConf.setBoolVar(conf, ConfVars.HIVE_STATS_COL_AUTOGATHER, genericStats && computeColStats); + String compactionQuery = (ci.partName == null) ? buildFullTableCompactionQuery(compactTableName, conf, icebergTable, columnsList, fileSizePredicate, orderBy) : diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java index 2d2992544397..65b67346888f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java @@ -35,6 +35,7 @@ import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.Table; import org.apache.iceberg.mr.hive.IcebergTableUtil; +import org.apache.iceberg.mr.hive.compaction.IcebergCompactionService; /** * What a write does to a table's column statistics. Iceberg keeps them in one file that is always @@ -66,7 +67,9 @@ record ColStatsWrite( boolean analyze, boolean analyzePartition, boolean compaction, - boolean fullTableMajorCompaction, + boolean majorCompaction, + boolean fullTableRewrite, + boolean singlePartitionRewrite, // what its commit did to the rows boolean holdsOnlyAddedRows, boolean emptySnapshot, @@ -94,8 +97,10 @@ public static ColStatsWritePolicy resolve( () -> IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf), isAnalyze(conf), isAnalyzePartition(conf), - SessionState.get() != null && SessionState.get().isCompaction(), - isFullTableMajorCompaction(conf), + isCompaction(), + isMajorCompaction(conf), + isFullTableRewrite(conf), + isSinglePartitionRewrite(conf), holdsOnlyAddedRows(snapshot), IcebergTableUtil.isEmptySnapshot(snapshot), wroteNoRows(conf), @@ -114,7 +119,8 @@ static ColStatsWritePolicy resolve(ColStatsWrite write) { } if (write.compaction()) { // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. - return write.fullTableMajorCompaction() && !write.isStatsAccurate() ? REPLACE : SKIP; + return write.fullTableRewrite() && write.majorCompaction() && !write.isStatsAccurate() ? + REPLACE : SKIP; } // ANALYZE reads the whole table, so it replaces. return write.analyze() ? REPLACE : resolveForWrite(write); @@ -126,6 +132,10 @@ private static ColStatsWritePolicy resolveForPartitions(ColStatsWrite write) { // what it read is what it named, not what the snapshot it reads happens to hold return write.analyzePartition() ? MERGE : REPLACE; } + if (write.compaction()) { + // a partition read whole describes itself; part of one describes none of it + return write.singlePartitionRewrite() && write.majorCompaction() ? MERGE : SKIP; + } if (write.holdsOnlyAddedRows() || write.emptySnapshot()) { return REPLACE; } @@ -185,9 +195,22 @@ private static boolean isAnalyzePartition(Configuration conf) { return SessionStateUtil.getResource(conf, ColumnStatsSemanticAnalyzer.ANALYZE_PARTITION).isPresent(); } - /** Whether the compaction read the whole table: a minor one sets a file size threshold. */ - private static boolean isFullTableMajorCompaction(Configuration conf) { - return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)) && - conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + private static boolean isCompaction() { + return SessionState.get() != null && SessionState.get().isCompaction(); + } + + /** Whether the compaction read every file of what it was pointed at: a minor one skips by size. */ + private static boolean isMajorCompaction(Configuration conf) { + return conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + } + + /** Whether what it was pointed at was the whole table, which only an unpartitioned one is. */ + private static boolean isFullTableRewrite(Configuration conf) { + return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)); + } + + /** Whether the compaction was pointed at one partition: a spec-evolution one carries PARTITION too. */ + private static boolean isSinglePartitionRewrite(Configuration conf) { + return conf.get(IcebergCompactionService.PARTITION_PATH) != null; } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 3b5a9fca8c79..0320adca9bb2 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; @@ -50,6 +51,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatistics; import org.apache.iceberg.Schema; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; @@ -61,6 +63,7 @@ import org.apache.iceberg.mr.hive.test.utils.HiveIcebergTestUtils; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; @@ -808,6 +811,31 @@ public void testColumnStatsAccurate() throws Exception { } } + @Test + public void testTableLevelColStatsTakeInWhatEachInsertAdds() { + // statistics kept for the table as a whole take in what a write adds to it, so a table + // described in full stays described as rows arrive + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_incr"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertTrue(colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (100, 'c')"); + + Assert.assertTrue("the table stays described, having taken the write in", + colStatsAccurate(identifier)); + checkColStatMinMaxValue(identifier.name(), "id", 1, 100); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL.varname, true); + } + @Test public void testMergeStatsWithInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); @@ -1710,6 +1738,264 @@ public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { Assert.assertFalse(colStatsAccurate(identifier)); } + @Test + public void testCarriedPartitionColStatsAreAnchoredByWhetherTheyStillHold() { + // ANALYZE full table -> DML on two partitions -> ANALYZE one of them. The partition the DML + // never touched is carried into the new file; the one it reached is not, having stopped + // describing itself. + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_anchor"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + + // one write reaching p=a and p=b, leaving p=c alone + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'a'), (9, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + icebergTable.refresh(); + Assert.assertEquals("the write reached p=b, so what was stored for it is not carried", + ImmutableSet.of("p=a", "p=c"), colStatsPartitions(icebergTable)); + Assert.assertEquals("everything carried is described by the file being written", + icebergTable.currentSnapshot().snapshotId(), currentColStatsFile(icebergTable).snapshotId()); + Assert.assertEquals("and each of them still describes its partition", + ImmutableMap.of("p=a", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testEvolvedFromUnpartitionedDescribesTheSyntheticPartition() throws Exception { + // rows written before the table was partitioned belong to a partition of their own, and both + // the list a scan prunes to and the statistics stored have to name it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_void"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (100, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + + // the pruning path asks for every spec, which is how a scan reaches the legacy rows + List pruned = storageHandler().getPartitions(hmsTable).stream() + .map(org.apache.hadoop.hive.ql.metadata.Partition::getName) + .toList(); + Assert.assertTrue("the pruned list has to name the legacy rows' partition: " + pruned, + pruned.contains(DummyPartition.VOID)); + Assert.assertTrue("and the statistics have to describe it: " + colStatsPartitions(icebergTable), + colStatsPartitions(icebergTable).contains(DummyPartition.VOID)); + + // a write to one partition and an ANALYZE naming only that one: the rest are carried + shell.executeStatement("INSERT INTO " + identifier + " VALUES (7, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + icebergTable.refresh(); + + Assert.assertTrue("the legacy rows' partition has to survive an ANALYZE that never named it: " + + colStatsPartitions(icebergTable), + colStatsPartitions(icebergTable).contains(DummyPartition.VOID)); + } + + @Test + public void testPartitionColStatsSurviveAWriteToAnotherPartition() { + // a write reaches one partition, and the ones it never touched still describe themselves + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_untouched"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals(ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), + upToDateByPartition(icebergTable)); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE p = 'a'"); + icebergTable.refresh(); + Assert.assertEquals("only the partition the delete reached stops describing itself", + ImmutableMap.of("p=a", false, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testPartitionColStatsGoStaleOnEveryFormatVersion() { + // a version 1 table numbers every snapshot 0, so sequence numbers cannot order them; where + // they sit on the ancestry can, and that is what every version is read by + assumeParquetHiveCatalogIceberg(); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + + for (int version = 1; version <= 3; version++) { + TableIdentifier identifier = TableIdentifier.of("default", "orders_format_v" + version); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='" + version + "')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + if (version == 1) { + Assert.assertEquals("every snapshot of a version 1 table shares one sequence number", + 0, icebergTable.currentSnapshot().sequenceNumber()); + } + Assert.assertEquals("version " + version + ": what was just computed describes the table", + ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'b')"); + icebergTable.refresh(); + Assert.assertEquals("version " + version + ": only the partition the insert reached stops " + + "describing itself", + ImmutableMap.of("p=a", true, "p=b", false, "p=c", true), upToDateByPartition(icebergTable)); + } + } + + @Test + public void testAggrColStatsCountsOnlyPartitionsCarryingEveryColumnAsked() throws Exception { + // a partition described for only some of the columns asked about does not count as found + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_added_column"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + List partNames = ImmutableList.of("p=a", "p=b"); + Assert.assertEquals("both partitions carry the column asked about", 2, + storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("id"), partNames) + .getPartsFound()); + + // a column added after the statistics were stored is described by no entry of theirs + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (amount bigint)"); + AggrStats aggrStats = storageHandler().getAggrColStatsFor( + hmsTable(identifier), ImmutableList.of("id", "amount"), partNames); + Assert.assertEquals("a partition missing one of the columns asked about is not counted", + 0, aggrStats.getPartsFound()); + } + + @Test + public void testTableLevelColStatsAreWithheldWhenAWriteLeftThemBehind() { + // judge the granularity served: a current per partition file says nothing about whole-table + // numbers an older snapshot holds + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_granularities"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertFalse("whole-table numbers were stored and nothing has happened since", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Assert.assertTrue("the insert left the whole-table numbers behind, per partition ones aside", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + } + + @Test + public void testRowPreservingCommitsDoNotSpendTheSnapshotLookback() { + // the lookback bounds manifest reads, and a commit that moves no rows costs none of it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_rewritten"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setIntVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK, 1); + Table icebergTable = testTables.loadTable(identifier); + for (int rewrite = 0; rewrite < 3; rewrite++) { + icebergTable.rewriteManifests().clusterBy(file -> "all").commit(); + } + icebergTable.refresh(); + Assert.assertEquals(DataOperations.REPLACE, icebergTable.currentSnapshot().operation()); + Assert.assertEquals("more rewrites than the lookback allows still leave the statistics placed", + ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testUpdateStalesThePartitionItLeftAndTheOneItReached() { + // moving a row between partitions deletes from one and writes to the other, and afterwards + // neither of them describes itself + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_moved"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals(ImmutableMap.of("p=a", true, "p=b", true, "p=c", true), + upToDateByPartition(icebergTable)); + + shell.executeStatement("UPDATE " + identifier + " SET p = 'b' WHERE id = 1"); + icebergTable.refresh(); + Assert.assertEquals("the partition it took the row from and the one it put it in both changed", + ImmutableMap.of("p=a", false, "p=b", false, "p=c", true), upToDateByPartition(icebergTable)); + } + + @Test + public void testCopyOnWriteDeleteStalesOnlyTheRewrittenPartition() { + // a copy-on-write delete rewrites the files of the partition it touches rather than adding + // delete files, so the walk has to read what the snapshot replaced, not only what it removed + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_cow"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2', 'write.delete.mode'='copy-on-write')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b'), (3, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + shell.executeStatement("DELETE FROM " + identifier + " WHERE id = 1"); + Table icebergTable = testTables.loadTable(identifier); + Assert.assertEquals("only the partition whose files were rewritten stops describing itself", + ImmutableMap.of("p=a", false, "p=b", true, "p=c", true), upToDateByPartition(icebergTable)); + } + + /** The statistics file a read of the table serves. */ + private StatisticsFile currentColStatsFile(Table icebergTable) { + return IcebergTableUtil.findColStatsFile( + icebergTable, icebergTable.currentSnapshot().snapshotId(), shell.getHiveConf()); + } + + /** The partitions the current statistics file describes, whether or not they still hold. */ + private Set colStatsPartitions(Table icebergTable) { + return currentColStatsFile(icebergTable).blobMetadata().stream() + .map(metadata -> metadata.properties().get(IcebergTableUtil.PARTITION_FIELD)) + .collect(Collectors.toSet()); + } + + /** Whether each stored partition's statistics still describe it at the current snapshot. */ + private Map upToDateByPartition(Table icebergTable) { + StatisticsFile statsFile = currentColStatsFile(icebergTable); + Predicate upToDate = IcebergTableUtil.upToDateColStats( + icebergTable, icebergTable.currentSnapshot(), statsFile, shell.getHiveConf(), true); + return statsFile.blobMetadata().stream().collect(Collectors.toMap( + metadata -> metadata.properties().get(IcebergTableUtil.PARTITION_FIELD), + metadata -> upToDate.test(metadata.properties().get(IcebergTableUtil.PARTITION_FIELD)), + (first, second) -> second)); + } + private boolean colStatsAccurate(TableIdentifier identifier) { return StatsSetupConst.areColumnStatsUptoDate(hmsTable(identifier).getParameters(), "id"); } @@ -1769,6 +2055,9 @@ public void testColStatsAfterPartitionTruncate() { shell.executeStatement("TRUNCATE TABLE " + identifier + " PARTITION (p = 'a')"); Assert.assertFalse(colStatsAccurate(identifier)); + // the emptied partition describes rows it no longer holds, and the one beside it is untouched + Assert.assertEquals(ImmutableMap.of("p=a", false, "p=b", true), + upToDateByPartition(testTables.loadTable(identifier))); shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); Assert.assertFalse(colStatsAccurate(identifier)); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java index f118951b6ddc..7ab1f978f95b 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java @@ -80,6 +80,21 @@ public void onlyAWholeTableCompactionRefreshesStaleStatistics() { assertEquals(REPLACE, resolve(tableWideStats().fullTableMajorCompaction().build())); assertEquals(SKIP, resolve(tableWideStats().compaction().build())); assertEquals(SKIP, resolve(tableWideStats().fullTableMajorCompaction().statsAccurate().build())); + // a partitioned table's compaction never rewrites the whole table, so it cannot refresh it + assertEquals(SKIP, resolve(tableWideStats().majorCompaction().build())); + } + + @Test + public void aCompactionSubstitutesOnlyThePartitionsItReadWhole() { + // the numbers of a partition it rewrote entirely stand for it, whatever it held before + assertEquals(MERGE, resolve(partitionLevel().majorCompaction().singlePartitionRewrite().build())); + assertEquals(MERGE, + resolve(partitionLevel().majorCompaction().singlePartitionRewrite().statsAccurate().build())); + // one that skipped files by size measured part of a partition, which describes none of it + assertEquals(SKIP, resolve(partitionLevel().compaction().singlePartitionRewrite().build())); + // and one clearing an older spec rewrites into partitions holding rows it never read + assertEquals(SKIP, resolve(partitionLevel().majorCompaction().build())); + assertEquals(SKIP, resolve(partitionLevel().compaction().build())); } @Test @@ -120,7 +135,9 @@ private static final class Builder { private boolean analyze; private boolean analyzePartition; private boolean compaction; - private boolean fullTableMajorCompaction; + private boolean majorCompaction; + private boolean fullTableRewrite; + private boolean singlePartitionRewrite; private boolean holdsOnlyAddedRows; private boolean emptySnapshot; private boolean wroteNoRows; @@ -153,8 +170,21 @@ private Builder compaction() { return this; } + /** A compaction that read every file of what it was pointed at. */ + private Builder majorCompaction() { + majorCompaction = true; + return compaction(); + } + + /** A major compaction pointed at the whole table, which only an unpartitioned one is. */ private Builder fullTableMajorCompaction() { - fullTableMajorCompaction = true; + fullTableRewrite = true; + return majorCompaction(); + } + + /** A compaction pointed at one named partition, rather than at what an older spec left. */ + private Builder singlePartitionRewrite() { + singlePartitionRewrite = true; return compaction(); } @@ -190,8 +220,8 @@ private Builder statsAccurate() { private ColStatsWrite build() { return new ColStatsWrite(tableWideStats, keepsStatsPerPartition, partitioned, () -> statsAccurate, - analyze, analyzePartition, compaction, fullTableMajorCompaction, holdsOnlyAddedRows, - emptySnapshot, wroteNoRows, replacePartitions); + analyze, analyzePartition, compaction, majorCompaction, fullTableRewrite, + singlePartitionRewrite, holdsOnlyAddedRows, emptySnapshot, wroteNoRows, replacePartitions); } } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q new file mode 100644 index 000000000000..dd16808061a1 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q @@ -0,0 +1,60 @@ +-- what ANALYZE stores for a table whose partitioning changed under it +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the inserts must leave the stored statistics alone for what happens to them to be visible +set hive.stats.column.autogather=false; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +-- these rows belong to the synthetic partition, the one an unpartitioned spec names +insert into ice_evo values (1, 'a'), (100, 'b'); + +alter table ice_evo set partition spec (p); + +insert into ice_evo values (5, 'a'); + +-- a full table ANALYZE reads every partition of every spec the table holds +analyze table ice_evo compute statistics for columns; + +explain select id from ice_evo where id > 0; + +-- a write reaching one partition leaves the others describing themselves +insert into ice_evo values (7, 'a'); + +explain select id from ice_evo where id > 0; + +-- naming that partition measures it again; the ones it never named are carried +analyze table ice_evo partition (p='a') compute statistics for columns; + +explain select id from ice_evo where id > 0; + +select count(*) from ice_evo; + +drop table ice_evo; + +-- a table described in full, then partitioned differently and written to: what the new spec's +-- partitions hold is not described until an ANALYZE reads them +create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_evo2 values (1, 'a'), (7, 'b'); +analyze table ice_evo2 compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +alter table ice_evo2 set partition spec (p, truncate(1, p)); + +insert into ice_evo2 values (9, 'a'); + +explain select id from ice_evo2 where id > 0; + +analyze table ice_evo2 compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +drop table ice_evo2; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q new file mode 100644 index 000000000000..66eccc02efab --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_staleness.q @@ -0,0 +1,59 @@ +-- what a write leaves of the column statistics stored before it, at each granularity +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the writes below must leave the stored statistics alone for what happens to them to be visible +set hive.stats.column.autogather=false; + +-- an unpartitioned table keeps one set for the whole of it, and a write leaves none of it standing +create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_unpart values (1, 'a'), (2, 'b'); +analyze table ice_stale_unpart compute statistics for columns; + +explain select id from ice_stale_unpart where id > 0; + +insert into ice_stale_unpart values (3, 'c'); + +explain select id from ice_stale_unpart where id > 0; + +drop table ice_stale_unpart; + +-- a partitioned table asked for whole-table statistics keeps one set too, on the same terms +set hive.iceberg.stats.collect.partlevel=false; + +create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c'); +analyze table ice_stale_tbllevel compute statistics for columns; + +explain select id from ice_stale_tbllevel where id > 0; + +insert into ice_stale_tbllevel values (9, 'a'); + +explain select id from ice_stale_tbllevel where id > 0; + +drop table ice_stale_tbllevel; + +-- kept per partition, only the partitions a write reached stop describing themselves, and what +-- the rest still describe is a part of what the scan reads +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd'); +analyze table ice_stale_partlevel compute statistics for columns; + +explain select id from ice_stale_partlevel where id > 0; + +insert into ice_stale_partlevel values (9, 'a'), (9, 'b'); + +explain select id from ice_stale_partlevel where id > 0; + +drop table ice_stale_partlevel; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q new file mode 100644 index 000000000000..0abb64656544 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_write_paths.q @@ -0,0 +1,51 @@ +-- what each kind of write leaves describing the table +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); +insert into ice_src values (1, 'a'), (7, 'b'); + +-- a CREATE TABLE AS reads every row it writes, so it describes all of them +create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src; + +explain select id from ice_ctas where id > 0; + +drop table ice_ctas; + +-- an insert gathers as it writes, so an unpartitioned table stays described +create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_unpart_w values (1, 'a'), (7, 'b'); +analyze table ice_unpart_w compute statistics for columns; + +explain select id from ice_unpart_w where id > 0; + +insert into ice_unpart_w values (9, 'c'); + +explain select id from ice_unpart_w where id > 0; + +drop table ice_unpart_w; + +-- an overwrite of the whole table replaces the rows and what described them alike +create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_iow values (1, 'a'), (7, 'b'); +analyze table ice_iow compute statistics for columns; + +explain select id from ice_iow where id > 0; + +insert overwrite table ice_iow select id + 100, p from ice_src; + +explain select id from ice_iow where id > 0; + +drop table ice_iow; +drop table ice_src; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q new file mode 100644 index 000000000000..8fa963997b79 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_compaction_colstats_compute.q @@ -0,0 +1,55 @@ +-- what a compaction leaves behind for a table that had no column statistics +set hive.explain.user=false; +-- a fetch task prints no statistics, and it is the statistics this is about +set hive.fetch.task.conversion=none; +set hive.stats.autogather=true; +-- the inserts must gather nothing, so that what the compaction gathers is what shows +set hive.stats.column.autogather=false; +set hive.iceberg.stats.collect.partlevel=true; + +-- a major compaction of an unpartitioned table reads every row it holds, so what it measures +-- describes the whole of it +create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500'); + +insert into ice_comp_unpart values (1, 'a'); +insert into ice_comp_unpart values (2, 'a'); +insert into ice_comp_unpart values (3, 'a'); +insert into ice_comp_unpart values (7, 'b'); + +explain select id from ice_comp_unpart where id > 0; + +alter table ice_comp_unpart COMPACT 'major' and wait; + +explain select id from ice_comp_unpart where id > 0; + +select min(id), max(id) from ice_comp_unpart; + +drop table ice_comp_unpart; + +-- a major compaction of one partition reads every row of that partition, and none of the others, +-- so it describes that partition alone +create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true'); + +insert into ice_comp values (1, 'a'); +insert into ice_comp values (2, 'a'); +insert into ice_comp values (3, 'a'); +insert into ice_comp values (4, 'a'); +insert into ice_comp values (7, 'b'); + +explain select id from ice_comp where id > 0; + +alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait; + +explain select id from ice_comp where id > 0; + +select min(id), max(id) from ice_comp; + +drop table ice_comp; diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out index 5ba761a21709..55cd325ea776 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out @@ -296,129 +296,129 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: a (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: c (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized Reducer 10 @@ -430,13 +430,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 7 Data size: 1436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 1436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 216 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 11 Reduce Operator Tree: @@ -447,23 +447,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 8 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -483,26 +483,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -512,13 +512,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1230 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 14 Reduce Operator Tree: @@ -529,11 +529,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 6 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1353 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -544,13 +544,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 6 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 1852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1431 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 16 Reduce Operator Tree: @@ -561,16 +561,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 17 Execution mode: vectorized @@ -578,7 +578,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -597,17 +597,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1574 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 787 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -620,31 +620,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 19 Execution mode: vectorized @@ -653,11 +653,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -668,11 +668,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 7 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 7 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized @@ -681,11 +681,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 21 Execution mode: vectorized @@ -694,11 +694,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 22 Execution mode: vectorized @@ -707,31 +707,31 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 23 Execution mode: vectorized @@ -740,16 +740,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 24 Execution mode: vectorized @@ -758,11 +758,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -773,17 +773,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 7 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -794,11 +794,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 8 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 8 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1532 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: @@ -809,24 +809,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1658 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 8 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 8 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 8 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1381 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: @@ -837,14 +837,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 1519 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2306 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -859,13 +859,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 126 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 9 Reduce Operator Tree: @@ -876,11 +876,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 7 Data size: 1380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 7 Data size: 1380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 138 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Union 7 Vertex: Union 7 @@ -988,58 +988,58 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 13 @@ -1071,20 +1071,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1104,26 +1104,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reducer 12 Execution mode: vectorized Reduce Operator Tree: @@ -1131,17 +1131,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -1152,11 +1152,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 2 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 2 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1167,24 +1167,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 2 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1195,14 +1195,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1217,13 +1217,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized @@ -1231,7 +1231,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1250,17 +1250,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1275,42 +1275,42 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reducer 9 Execution mode: vectorized Reduce Operator Tree: @@ -1318,11 +1318,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1426,58 +1426,58 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 13 @@ -1509,20 +1509,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1542,26 +1542,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reducer 12 Execution mode: vectorized Reduce Operator Tree: @@ -1569,11 +1569,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1584,11 +1584,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1599,24 +1599,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) - Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1627,14 +1627,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1649,13 +1649,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized @@ -1663,7 +1663,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1682,17 +1682,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1707,62 +1707,62 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col1) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reducer 9 Execution mode: vectorized Reduce Operator Tree: @@ -1770,17 +1770,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Union 5 Vertex: Union 5 diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out index 6e11edce5a7e..d8f7ecf10e8d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out @@ -84,54 +84,54 @@ Stage-4 <-Reducer 2 [CONTAINS] File Output Operator [FS_46] table:{"name:":"default.ice01"} - Select Operator [SEL_44] (rows=3 width=266) + Select Operator [SEL_44] (rows=7 width=206) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_43] (rows=3 width=266) + Merge Join Operator [MERGEJOIN_43] (rows=7 width=206) Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_59] PartitionCols:_col4 - Select Operator [SEL_56] (rows=6 width=280) + Select Operator [SEL_56] (rows=7 width=188) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_53] (rows=6 width=76) + Filter Operator [FIL_53] (rows=7 width=188) predicate:((((id = 2) or (id > 4)) is null or (id < 2) or ((id > 2) and (id <= 4))) and FILE__PATH is not null) - TableScan [TS_0] (rows=7 width=78) - default@ice01,ice01,Tbl:COMPLETE,Col:COMPLETE,Output:["id","name"] + TableScan [TS_0] (rows=7 width=188) + default@ice01,ice01,Tbl:COMPLETE,Col:NONE,Output:["id","name"] <-Reducer 4 [SIMPLE_EDGE] vectorized SHUFFLE [RS_65] PartitionCols:_col0 - Group By Operator [GBY_64] (rows=3 width=184) + Group By Operator [GBY_64] (rows=3 width=4) Output:["_col0"],keys:_col0 - Select Operator [SEL_63] (rows=3 width=184) + Select Operator [SEL_63] (rows=3 width=4) Output:["_col0"] - Filter Operator [FIL_62] (rows=3 width=184) + Filter Operator [FIL_62] (rows=3 width=4) predicate:(row_number_window_0 = 1) - PTF Operator [PTF_61] (rows=6 width=184) + PTF Operator [PTF_61] (rows=7 width=4) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] - Select Operator [SEL_60] (rows=6 width=184) + Select Operator [SEL_60] (rows=7 width=4) Output:["_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_57] PartitionCols:FILE__PATH - Filter Operator [FIL_54] (rows=6 width=4) + Filter Operator [FIL_54] (rows=7 width=4) predicate:(((id = 2) or (id > 4)) and FILE__PATH is not null) Please refer to the previous TableScan [TS_0] <-Reducer 5 [CONTAINS] vectorized File Output Operator [FS_70] table:{"name:":"default.ice01"} - Select Operator [SEL_69] (rows=3 width=266) + Select Operator [SEL_69] (rows=3 width=188) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_68] (rows=3 width=258) + Filter Operator [FIL_68] (rows=3 width=188) predicate:(row_number_window_0 = 1) - PTF Operator [PTF_67] (rows=6 width=272) + PTF Operator [PTF_67] (rows=7 width=188) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] - Select Operator [SEL_66] (rows=6 width=272) + Select Operator [SEL_66] (rows=7 width=188) Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_58] PartitionCols:FILE__PATH - Filter Operator [FIL_55] (rows=6 width=76) - predicate:((id > 4) or (id = 2)) + Filter Operator [FIL_55] (rows=7 width=188) + predicate:((id = 2) or (id > 4)) Please refer to the previous TableScan [TS_0] PREHOOK: query: delete from ice01 where id>4 OR id=2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out index 0ace027030fa..fba84461503d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_semijoin_reduction.q.out @@ -80,20 +80,20 @@ STAGE PLANS: TableScan alias: srcpart_date_n7 filterExpr: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 349784 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_n3_key1_min) AND DynamicValue(RS_7_srcpart_small_n3_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_n3_key1_bloom_filter))) (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1900 Data size: 332294 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 4 Map Operator Tree: @@ -138,17 +138,17 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 126 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2090 Data size: 365523 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized @@ -157,10 +157,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out new file mode 100644 index 000000000000..fdcb07b04a1c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out @@ -0,0 +1,433 @@ +PREHOOK: query: create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo +POSTHOOK: query: create external table ice_evo (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo +PREHOOK: query: insert into ice_evo values (1, 'a'), (100, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (1, 'a'), (100, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: alter table ice_evo set partition spec (p) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_evo +POSTHOOK: query: alter table ice_evo set partition spec (p) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +PREHOOK: query: insert into ice_evo values (5, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (5, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: analyze table ice_evo compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: default@ice_evo +PREHOOK: Output: default@ice_evo@__NO_PARTITION__ +PREHOOK: Output: default@ice_evo@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +POSTHOOK: Output: default@ice_evo@__NO_PARTITION__ +POSTHOOK: Output: default@ice_evo@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_evo values (7, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo +POSTHOOK: query: insert into ice_evo values (7, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table ice_evo partition (p='a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: default@ice_evo +PREHOOK: Output: default@ice_evo@__NO_PARTITION__ +PREHOOK: Output: default@ice_evo@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo partition (p='a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: default@ice_evo +POSTHOOK: Output: default@ice_evo@__NO_PARTITION__ +POSTHOOK: Output: default@ice_evo@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +4 +PREHOOK: query: drop table ice_evo +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_evo +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo +POSTHOOK: query: drop table ice_evo +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo +PREHOOK: query: create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: create external table ice_evo2 (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: insert into ice_evo2 values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: insert into ice_evo2 values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: analyze table ice_evo2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: alter table ice_evo2 set partition spec (p, truncate(1, p)) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_evo2 +POSTHOOK: query: alter table ice_evo2 set partition spec (p, truncate(1, p)) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: insert into ice_evo2 values (9, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: insert into ice_evo2 values (9, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_evo2 +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table ice_evo2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +PREHOOK: Output: default@ice_evo2@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +POSTHOOK: Output: default@ice_evo2@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_evo2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_evo2 +POSTHOOK: query: drop table ice_evo2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_evo2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out index 347519552cf2..f8711b175f14 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out @@ -156,14 +156,14 @@ STAGE PLANS: TableScan alias: ice_p filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: id (type: int), 'a' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out new file mode 100644 index 000000000000..76cedce6ceca --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_staleness.q.out @@ -0,0 +1,426 @@ +PREHOOK: query: create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: create external table ice_stale_unpart (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: insert into ice_stale_unpart values (1, 'a'), (2, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: insert into ice_stale_unpart values (1, 'a'), (2, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: analyze table ice_stale_unpart compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_unpart compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_unpart values (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: insert into ice_stale_unpart values (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: explain select id from ice_stale_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_unpart +POSTHOOK: query: drop table ice_stale_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_unpart +PREHOOK: query: create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: create external table ice_stale_tbllevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: insert into ice_stale_tbllevel values (1, 'a'), (7, 'b'), (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: analyze table ice_stale_tbllevel compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: default@ice_stale_tbllevel +PREHOOK: Output: default@ice_stale_tbllevel@p=a +PREHOOK: Output: default@ice_stale_tbllevel@p=b +PREHOOK: Output: default@ice_stale_tbllevel@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_tbllevel compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: Output: default@ice_stale_tbllevel@p=a +POSTHOOK: Output: default@ice_stale_tbllevel@p=b +POSTHOOK: Output: default@ice_stale_tbllevel@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_tbllevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_tbllevel values (9, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: insert into ice_stale_tbllevel values (9, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_tbllevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_tbllevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_tbllevel +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_tbllevel +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_tbllevel +POSTHOOK: query: drop table ice_stale_tbllevel +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_tbllevel +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_tbllevel +PREHOOK: query: create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: create external table ice_stale_partlevel (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: insert into ice_stale_partlevel values (1, 'a'), (7, 'b'), (3, 'c'), (5, 'd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: analyze table ice_stale_partlevel compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: default@ice_stale_partlevel +PREHOOK: Output: default@ice_stale_partlevel@p=a +PREHOOK: Output: default@ice_stale_partlevel@p=b +PREHOOK: Output: default@ice_stale_partlevel@p=c +PREHOOK: Output: default@ice_stale_partlevel@p=d +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_stale_partlevel compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: default@ice_stale_partlevel +POSTHOOK: Output: default@ice_stale_partlevel@p=a +POSTHOOK: Output: default@ice_stale_partlevel@p=b +POSTHOOK: Output: default@ice_stale_partlevel@p=c +POSTHOOK: Output: default@ice_stale_partlevel@p=d +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_stale_partlevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_partlevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_partlevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_stale_partlevel values (9, 'a'), (9, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: insert into ice_stale_partlevel values (9, 'a'), (9, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_stale_partlevel +PREHOOK: query: explain select id from ice_stale_partlevel where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_stale_partlevel where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_stale_partlevel + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_stale_partlevel +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_stale_partlevel +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_stale_partlevel +POSTHOOK: query: drop table ice_stale_partlevel +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_stale_partlevel +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_stale_partlevel diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out new file mode 100644 index 000000000000..4a35daa7f9af --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_write_paths.q.out @@ -0,0 +1,377 @@ +PREHOOK: query: create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_src +POSTHOOK: query: create external table ice_src (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_src +PREHOOK: query: insert into ice_src values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_src +POSTHOOK: query: insert into ice_src values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_src +PREHOOK: query: create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@ice_src +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: create external table ice_ctas +stored by iceberg tblproperties ('format-version'='2') as select * from ice_src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: Lineage: ice_ctas.id SIMPLE [(ice_src)ice_src.FieldSchema(name:id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_ctas.p SIMPLE [(ice_src)ice_src.FieldSchema(name:p, type:string, comment:null), ] +PREHOOK: query: explain select id from ice_ctas where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_ctas +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_ctas where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_ctas +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_ctas + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_ctas +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_ctas +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_ctas +POSTHOOK: query: drop table ice_ctas +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_ctas +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_ctas +PREHOOK: query: create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: create external table ice_unpart_w (id bigint, p string) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: insert into ice_unpart_w values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: insert into ice_unpart_w values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: analyze table ice_unpart_w compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_unpart_w compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_unpart_w where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_unpart_w where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart_w + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert into ice_unpart_w values (9, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: insert into ice_unpart_w values (9, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: explain select id from ice_unpart_w where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_unpart_w where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart_w + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_unpart_w +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_unpart_w +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart_w +POSTHOOK: query: drop table ice_unpart_w +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_unpart_w +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart_w +PREHOOK: query: create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_iow +POSTHOOK: query: create external table ice_iow (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_iow +PREHOOK: query: insert into ice_iow values (1, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_iow +POSTHOOK: query: insert into ice_iow values (1, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_iow +PREHOOK: query: analyze table ice_iow compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_iow +PREHOOK: Output: default@ice_iow +PREHOOK: Output: default@ice_iow@p=a +PREHOOK: Output: default@ice_iow@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_iow compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: default@ice_iow +POSTHOOK: Output: default@ice_iow@p=a +POSTHOOK: Output: default@ice_iow@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_iow where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_iow +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_iow where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_iow + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: insert overwrite table ice_iow select id + 100, p from ice_src +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_src +PREHOOK: Output: default@ice_iow +POSTHOOK: query: insert overwrite table ice_iow select id + 100, p from ice_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: default@ice_iow +PREHOOK: query: explain select id from ice_iow where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_iow +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_iow where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_iow + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table ice_iow +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_iow +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_iow +POSTHOOK: query: drop table ice_iow +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_iow +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_iow +PREHOOK: query: drop table ice_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_src +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_src +POSTHOOK: query: drop table ice_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_src diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 1e128a396b6d..8b4963ceb7e7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -164,14 +164,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -180,17 +180,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -200,14 +200,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -849,32 +849,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 6 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), country (type: string), 'TX' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -886,7 +886,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 6 Data size: 1626 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -899,14 +899,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -970,32 +970,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), age (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.9 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1007,7 +1007,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 10 Data size: 2730 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3720 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1020,14 +1020,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1236 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1091,32 +1091,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_string - Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: name (type: string), 54 (type: int), 'India' (type: string), state (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col3 (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string) outputColumnNames: name, age, country, state - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(length(name)), avg(COALESCE(length(name),0)), count(1), count(name), compute_bit_vector_hll(name), min(age), max(age), count(age), compute_bit_vector_hll(age), max(length(country)), avg(COALESCE(length(country),0)), count(country), compute_bit_vector_hll(country), max(length(state)), avg(COALESCE(length(state),0)), count(state), compute_bit_vector_hll(state) - minReductionHashAggr: 0.95 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: struct), _col11 (type: bigint), _col12 (type: binary), _col13 (type: int), _col14 (type: struct), _col15 (type: bigint), _col16 (type: binary) Execution mode: vectorized Reducer 2 @@ -1128,7 +1128,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 20 Data size: 5460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1141,14 +1141,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), max(VALUE._col9), avg(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12), max(VALUE._col13), avg(VALUE._col14), count(VALUE._col15), compute_bit_vector_hll(VALUE._col16) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col9,0)) (type: bigint), COALESCE(_col10,0) (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col13,0)) (type: bigint), COALESCE(_col14,0) (type: double), (_col2 - _col15) (type: bigint), COALESCE(ndv_compute_bit_vector(_col16),0) (type: bigint), _col16 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1062 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1467,14 +1467,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: bigintcol (type: bigint), intcol (type: int), DATE'1999-12-31' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1483,17 +1483,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1503,14 +1503,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1573,14 +1573,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 234675894076895090L (type: bigint), intcol (type: int), DATE'1999-12-26' (type: date) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1589,17 +1589,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: date), _col10 (type: date), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1609,14 +1609,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'LONG' (type: string), _col0 (type: bigint), _col1 (type: bigint), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DATE' (type: string), _col9 (type: date), _col10 (type: date), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1921,14 +1921,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 34567890123456787L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1937,17 +1937,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1957,14 +1957,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2027,14 +2027,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_bigint - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 12346577399277578L (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2043,17 +2043,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: bigint) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2063,14 +2063,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), _col9 (type: bigint), _col10 (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2375,14 +2375,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2391,17 +2391,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2411,14 +2411,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 660 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2481,14 +2481,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_double - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189D (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2497,17 +2497,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: double) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2517,14 +2517,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DOUBLE' (type: string), _col9 (type: double), _col10 (type: double), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2829,14 +2829,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: datecol (type: date), intcol (type: int), 3.14786 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2845,17 +2845,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 4 Data size: 688 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 240 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.75 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2865,14 +2865,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 868 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2935,14 +2935,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_decimal - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'2022-01-25' (type: date), intcol (type: int), 3.189 (type: decimal(10,6)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2951,17 +2951,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: date), _col1 (type: int), _col2 (type: decimal(10,6)) outputColumnNames: datecol, intcol, pcol - Statistics: Num rows: 6 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(datecol), max(datecol), count(1), count(datecol), compute_bit_vector_hll(datecol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -2971,14 +2971,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'DECIMAL' (type: string), _col9 (type: decimal(10,6)), _col10 (type: decimal(10,6)), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1099 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 812 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 95fda3f4c11b..5e29ac3cd1c4 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -1691,14 +1691,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'1999-12-13' (type: date), bigintcol (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1707,17 +1707,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 6 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.8333333 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1727,14 +1727,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 612 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1797,14 +1797,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_parquet_date_transform_day - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: DATE'1999-12-02' (type: date), 234675894076895090L (type: bigint), intcol (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1813,17 +1813,17 @@ STAGE PLANS: Select Operator expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date) outputColumnNames: bigintcol, intcol, pcol - Statistics: Num rows: 12 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol) - minReductionHashAggr: 0.9166667 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -1833,14 +1833,14 @@ STAGE PLANS: aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 'DATE' (type: string), _col0 (type: date), _col1 (type: date), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), _col5 (type: bigint), _col6 (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 888 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 604 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out new file mode 100644 index 000000000000..af2e3e249bd8 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_compaction_colstats_compute.q.out @@ -0,0 +1,342 @@ +PREHOOK: query: create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: create external table ice_comp_unpart (id bigint, p string) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (1, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (1, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (2, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (2, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (3, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (3, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: insert into ice_comp_unpart values (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: insert into ice_comp_unpart values (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: explain select id from ice_comp_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: alter table ice_comp_unpart COMPACT 'major' and wait +PREHOOK: type: ALTERTABLE_COMPACT +PREHOOK: Input: default@ice_comp_unpart +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: alter table ice_comp_unpart COMPACT 'major' and wait +POSTHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: Input: default@ice_comp_unpart +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: explain select id from ice_comp_unpart where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp_unpart where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp_unpart + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_comp_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +POSTHOOK: query: select min(id), max(id) from ice_comp_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp_unpart +#### A masked pattern was here #### +1 7 +PREHOOK: query: drop table ice_comp_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_comp_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp_unpart +POSTHOOK: query: drop table ice_comp_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_comp_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp_unpart +PREHOOK: query: create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp +POSTHOOK: query: create external table ice_comp (id bigint, p string) + partitioned by spec (p) +stored by iceberg stored as orc +tblproperties ('format-version'='2', 'compactor.threshold.target.size'='1500', + -- a compaction runs long after the session that queued it, so the granularity it keeps + -- statistics at is asked for the way the compactor takes any of its settings + 'compactor.hive.iceberg.stats.collect.partlevel'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (1, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (1, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (2, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (2, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (3, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (3, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (4, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (4, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: insert into ice_comp values (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_comp +POSTHOOK: query: insert into ice_comp values (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_comp +PREHOOK: query: explain select id from ice_comp where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait +PREHOOK: type: ALTERTABLE_COMPACT +PREHOOK: Input: default@ice_comp +PREHOOK: Output: default@ice_comp@p=a +POSTHOOK: query: alter table ice_comp PARTITION (p='a') COMPACT 'major' and wait +POSTHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: Input: default@ice_comp +POSTHOOK: Output: default@ice_comp@p=a +PREHOOK: query: explain select id from ice_comp where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: explain select id from ice_comp where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_comp + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_comp +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_comp +#### A masked pattern was here #### +POSTHOOK: query: select min(id), max(id) from ice_comp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_comp +#### A masked pattern was here #### +1 7 +PREHOOK: query: drop table ice_comp +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_comp +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_comp +POSTHOOK: query: drop table ice_comp +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_comp +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_comp diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index 05342b5039c9..07ba4289b8f6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -470,17 +470,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: target_ice - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Execution mode: vectorized Map 4 @@ -509,20 +509,20 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 12 Data size: 1216 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 9 Data size: 912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), _col6 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) - Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Execution mode: vectorized @@ -533,7 +533,7 @@ STAGE PLANS: File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 9 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 556df48415c2..7b1f159b780d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -284,18 +284,18 @@ Stage-3 <-Reducer 2 [SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_21] PartitionCols:_col4, _col5 - Select Operator [SEL_20] (rows=11 width=224) + Select Operator [SEL_20] (rows=11 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_19] (rows=11 width=224) + Limit [LIM_19] (rows=11 width=420) Number of rows:100000 <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_18] - Select Operator [SEL_17] (rows=11 width=224) + Select Operator [SEL_17] (rows=11 width=420) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Limit [LIM_16] (rows=11 width=224) + Limit [LIM_16] (rows=11 width=420) Number of rows:100000 - TableScan [TS_0] (rows=11 width=224) - default@llap_orders,llap_orders,Tbl:COMPLETE,Col:COMPLETE,Output:["orderid","quantity","itemid","tradets","p1","p2"] + TableScan [TS_0] (rows=11 width=420) + default@llap_orders,llap_orders,Tbl:COMPLETE,Col:NONE,Output:["orderid","quantity","itemid","tradets","p1","p2"] Reducer 4 vectorized File Output Operator [FS_29] Select Operator [SEL_28] (rows=1 width=1###) @@ -306,7 +306,7 @@ Stage-3 PARTITION_ONLY_SHUFFLE [RS_24] Group By Operator [GBY_23] (rows=1 width=1###) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24"],aggregations:["min(orderid)","max(orderid)","count(1)","count(orderid)","compute_bit_vector_hll(orderid)","min(quantity)","max(quantity)","count(quantity)","compute_bit_vector_hll(quantity)","min(itemid)","max(itemid)","count(itemid)","compute_bit_vector_hll(itemid)","min(tradets)","max(tradets)","count(tradets)","compute_bit_vector_hll(tradets)","max(length(p1))","avg(COALESCE(length(p1),0))","count(p1)","compute_bit_vector_hll(p1)","max(length(p2))","avg(COALESCE(length(p2),0))","count(p2)","compute_bit_vector_hll(p2)"] - Select Operator [SEL_22] (rows=11 width=224) + Select Operator [SEL_22] (rows=11 width=420) Output:["orderid","quantity","itemid","tradets","p1","p2"] Please refer to the previous Select Operator [SEL_20] diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out index 9ea7240bd41b..9fb8158d7879 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out @@ -327,151 +327,151 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator keys: c (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(c) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Filter Operator predicate: (a <= 5) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: a (type: int) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Group By Operator aggregations: count(), count(a) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 10 @@ -483,11 +483,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 - Statistics: Num rows: 11 Data size: 3496 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 11 Data size: 3496 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2297 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -498,24 +498,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 - Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 - Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2486 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 11 Data size: 3672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 11 Data size: 3300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 11 Data size: 3300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1657 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: @@ -526,14 +526,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -548,13 +548,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint) Reducer 14 Reduce Operator Tree: @@ -565,11 +565,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col5 - Statistics: Num rows: 10 Data size: 1968 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 1968 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean) Reducer 15 Reduce Operator Tree: @@ -580,13 +580,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2, _col3, _col5, _col6 - Statistics: Num rows: 10 Data size: 2048 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 10 Data size: 2048 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: bigint), _col5 (type: boolean), _col6 (type: bigint) Reducer 16 Reduce Operator Tree: @@ -597,23 +597,23 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col2, _col3, _col5, _col6, _col8 - Statistics: Num rows: 11 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col3 <> 0L) and _col5 is not null) or ((_col6 <> 0L) and _col8 is not null)) (type: boolean) - Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 3 Data size: 612 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col2 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -633,26 +633,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 158 Basic stats: COMPLETE Column stats: NONE Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -662,13 +662,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 189 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 19 Reduce Operator Tree: @@ -679,11 +679,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 10 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 207 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 2 Reduce Operator Tree: @@ -694,13 +694,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 20 Reduce Operator Tree: @@ -711,13 +711,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 10 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 10 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 324 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 21 Reduce Operator Tree: @@ -728,17 +728,17 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 11 Data size: 2528 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 3 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 927 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 356 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -751,16 +751,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 23 Execution mode: vectorized @@ -769,38 +769,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 24 Execution mode: vectorized @@ -809,11 +809,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 25 Execution mode: vectorized @@ -822,11 +822,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 26 Execution mode: vectorized @@ -835,38 +835,38 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 27 Execution mode: vectorized @@ -875,16 +875,16 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 28 Execution mode: vectorized @@ -893,11 +893,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 29 Execution mode: vectorized @@ -906,11 +906,11 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -921,11 +921,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2029 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 4 Reduce Operator Tree: @@ -936,13 +936,13 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 - Statistics: Num rows: 10 Data size: 3088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 10 Data size: 3088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2146 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 5 Reduce Operator Tree: @@ -953,16 +953,16 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 - Statistics: Num rows: 11 Data size: 3408 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) - Statistics: Num rows: 3 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 3 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 6 Execution mode: vectorized @@ -970,7 +970,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -989,17 +989,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2360 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1048 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1527 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 3226 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1014,11 +1014,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 9 Reduce Operator Tree: @@ -1029,17 +1029,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2089 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Union 7 Vertex: Union 7 @@ -1155,69 +1155,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Map 14 @@ -1247,17 +1247,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 11 Reduce Operator Tree: @@ -1268,20 +1268,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reducer 12 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1301,26 +1301,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reducer 13 Execution mode: vectorized Reduce Operator Tree: @@ -1328,11 +1328,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: @@ -1343,11 +1343,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: @@ -1358,24 +1358,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null) (type: boolean) - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: @@ -1386,14 +1386,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1408,14 +1408,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 311 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1430,13 +1430,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized @@ -1444,7 +1444,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1463,17 +1463,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 911 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1488,47 +1488,47 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 5 Vertex: Union 5 @@ -1661,69 +1661,69 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: FILE__PATH is not null (type: boolean) - Statistics: Num rows: 9 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 2628 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1692 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized Reducer 10 @@ -1735,20 +1735,20 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reducer 11 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1768,26 +1768,26 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -1797,13 +1797,13 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 13 Execution mode: vectorized @@ -1811,7 +1811,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1830,17 +1830,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1855,60 +1855,60 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col0) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1918,14 +1918,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1938,17 +1938,17 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reducer 6 Reduce Operator Tree: @@ -1959,11 +1959,11 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 10 Data size: 3008 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 7 Reduce Operator Tree: @@ -1974,24 +1974,24 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null) (type: boolean) - Statistics: Num rows: 10 Data size: 3168 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 10 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 8 Reduce Operator Tree: @@ -2002,14 +2002,14 @@ STAGE PLANS: 0 _col5 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -2022,11 +2022,11 @@ STAGE PLANS: aggregations: count(VALUE._col0), count(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Union 4 Vertex: Union 4 @@ -2099,14 +2099,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tbl_ice - Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), 'All' (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 855 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 855 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out index ccf73acb8669..a33dcfab92b3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out index 0a245752e303..66273d7216bb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out index fe5345d98e33..3e6f66ad196b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out @@ -53,8 +53,8 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tbl_ice -Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[66][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800) PREHOOK: type: QUERY PREHOOK: Input: default@tbl_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 3c43b9ec35f6..47dde0580b33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -325,9 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (CUSTOM_EDGE) + Map 1 <- Map 5 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -370,8 +371,8 @@ STAGE PLANS: hashTableImplementationType: OPTIMIZED outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50 input vertices: - 1 Map 4 - Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 5 + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col39 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col32 (type: int), _col38 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col37 (type: int), _col6 (type: int), _col49 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col33 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col46 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col34 (type: int), _col30 (type: int), _col40 (type: decimal(7,2)), _col35 (type: int), _col4 (type: int), _col47 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col42 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col28 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col29 (type: int), _col41 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col50 (type: decimal(7,2)), _col31 (type: int), _col36 (type: int), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48 @@ -379,14 +380,14 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [49, 20, 42, 48, 34, 22, 36, 8, 4, 11, 58, 47, 7, 59, 12, 43, 18, 15, 54, 56, 21, 9, 44, 40, 50, 45, 5, 57, 6, 14, 13, 53, 52, 2, 17, 38, 3, 16, 19, 35, 10, 39, 51, 1, 37, 60, 41, 46, 55] - Statistics: Num rows: 7 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint), 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), null (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 @@ -395,13 +396,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [34, 35, 36, 37, 30, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 61, 53, 54, 55, 56, 57, 58, 59, 60] selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 61:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -413,7 +414,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -422,25 +423,28 @@ STAGE PLANS: native: true projectedOutputColumnNums: [31, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 62, 53, 54, 55, 56, 57, 58, 59, 60] selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 62:decimal(7,2) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat - output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat - serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe - name: default.store_sales + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + keyColumns: 41:int, 64:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 31:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 62:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 40:int), SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 38:int)) predicate: (_col23 is null and _col46 is null and _col35 is null) (type: boolean) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 2451181 (type: int), _col43 (type: int), _col33 (type: int), _col36 (type: int), _col8 (type: int), _col26 (type: int), _col28 (type: int), _col12 (type: int), _col7 (type: int), _col21 (type: int), _col40 (type: int), _col9 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col29 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col37 (type: decimal(7,2)), _col34 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 @@ -449,7 +453,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [63, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 63:int - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) null sort order: zz @@ -457,12 +461,12 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 64:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int + keyColumns: 3:int, 65:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 63:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: @@ -470,7 +474,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col6 (type: string), _col39 (type: bigint), _col44 (type: bigint) outputColumnNames: _col4, _col6, _col39, _col44 @@ -478,7 +482,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [34, 36, 35, 37] - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() Group By Vectorization: @@ -490,10 +494,10 @@ STAGE PLANS: vectorProcessingMode: HASH projectedOutputColumnNums: [0] keys: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz @@ -505,7 +509,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: 4:bigint - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col4 (type: bigint) Execution mode: vectorized Map Vectorization: @@ -522,13 +526,13 @@ STAGE PLANS: includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint] - Map 4 + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] + Map 5 Map Operator Tree: TableScan alias: store_sales filterExpr: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] @@ -538,7 +542,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongScalar(col 0:int, val 2451181), SelectColumnIsNotNull(col 2:int), SelectColumnIsNotNull(col 3:int)) predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 @@ -547,7 +551,7 @@ STAGE PLANS: native: true projectedOutputColumnNums: [23, 24, 25, 26, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col6 (type: int), _col7 (type: int) null sort order: zz @@ -560,7 +564,7 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: @@ -608,13 +612,48 @@ STAGE PLANS: File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.store_sales Reducer 3 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true + reduceColumnNullOrder: zz + reduceColumnSortOrder: ++ + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 25 + dataColumns: KEY._col3:int, KEY.iceberg_bucket(_col2, 3):int, VALUE._col0:int, VALUE._col1:int, VALUE._col2:int, VALUE._col3:int, VALUE._col4:int, VALUE._col5:int, VALUE._col6:int, VALUE._col7:int, VALUE._col8:int, VALUE._col9:int, VALUE._col10:int, VALUE._col11:decimal(7,2)/DECIMAL_64, VALUE._col12:decimal(7,2)/DECIMAL_64, VALUE._col13:decimal(7,2)/DECIMAL_64, VALUE._col14:decimal(7,2)/DECIMAL_64, VALUE._col15:decimal(7,2)/DECIMAL_64, VALUE._col16:decimal(7,2)/DECIMAL_64, VALUE._col17:decimal(7,2)/DECIMAL_64, VALUE._col18:decimal(7,2)/DECIMAL_64, VALUE._col19:decimal(7,2)/DECIMAL_64, VALUE._col20:decimal(7,2)/DECIMAL_64, VALUE._col21:decimal(7,2)/DECIMAL_64, VALUE._col22:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: int), VALUE._col7 (type: int), VALUE._col8 (type: int), VALUE._col9 (type: int), VALUE._col10 (type: int), VALUE._col11 (type: decimal(7,2)), VALUE._col12 (type: decimal(7,2)), VALUE._col13 (type: decimal(7,2)), VALUE._col14 (type: decimal(7,2)), VALUE._col15 (type: decimal(7,2)), VALUE._col16 (type: decimal(7,2)), VALUE._col17 (type: decimal(7,2)), VALUE._col18 (type: decimal(7,2)), VALUE._col19 (type: decimal(7,2)), VALUE._col20 (type: decimal(7,2)), VALUE._col21 (type: decimal(7,2)), VALUE._col22 (type: decimal(7,2)), KEY._col3 (type: int), KEY.iceberg_bucket(_col2, 3) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col3, iceberg_bucket(_col2, 3) + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1] + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.store_sales + Reducer 4 Execution mode: vectorized Reduce Vectorization: enabled: true @@ -643,14 +682,14 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true predicateExpression: FilterLongColGreaterLongScalar(col 4:bigint, val 1) predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -659,13 +698,13 @@ STAGE PLANS: native: true projectedOutputColumnNums: [5] selectExpressions: VectorUDFAdaptor(cardinality_violation(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -846,9 +885,10 @@ POSTHOOK: Output: default@merge_tmp_table POSTHOOK: Output: default@store_sales POSTHOOK: Output: default@store_sales Vertex dependency in root stage -Map 1 <- Map 4 (CUSTOM_EDGE) +Map 1 <- Map 5 (CUSTOM_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) +Reducer 4 <- Map 1 (SIMPLE_EDGE) Stage-6 Stats Work{} @@ -859,53 +899,66 @@ Stage-6 Dependency Collection{} Stage-4 Reducer 2 vectorized - File Output Operator [FS_57] + File Output Operator [FS_59] table:{"name:":"default.store_sales"} - Select Operator [SEL_56] + Select Operator [SEL_58] Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_53] + SHUFFLE [RS_54] PartitionCols:_col3, iceberg_bucket(_col2, 3) - Select Operator [SEL_49] (rows=3 width=#Masked#) + Select Operator [SEL_50] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - Filter Operator [FIL_45] (rows=3 width=#Masked#) - predicate:(_col23 is null and _col46 is null and _col35 is null) - Select Operator [SEL_42] (rows=7 width=#Masked#) + Filter Operator [FIL_46] (rows=1 width=#Masked#) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) + Select Operator [SEL_44] (rows=5 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48"] - Map Join Operator [MAPJOIN_41] (rows=7 width=#Masked#) - BucketMapJoin:true,Conds:SEL_40._col1, _col2=RS_39._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] - <-Map 4 [CUSTOM_EDGE] vectorized - MULTICAST [RS_39] + Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) + BucketMapJoin:true,Conds:SEL_42._col1, _col2=RS_41._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] + <-Map 5 [CUSTOM_EDGE] vectorized + MULTICAST [RS_41] PartitionCols:_col6 - Select Operator [SEL_38] (rows=2 width=#Masked#) + Select Operator [SEL_40] (rows=2 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"] - Filter Operator [FIL_37] (rows=2 width=#Masked#) + Filter Operator [FIL_39] (rows=2 width=#Masked#) predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) TableScan [TS_2] (rows=2 width=#Masked#) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] - <-Select Operator [SEL_40] (rows=5 width=#Masked#) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] + <-Select Operator [SEL_42] (rows=5 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] TableScan [TS_0] (rows=5 width=#Masked#) default@ssv,s,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"] Reducer 3 vectorized File Output Operator [FS_61] + table:{"name:":"default.store_sales"} + Select Operator [SEL_60] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"] + <-Map 1 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_55] + PartitionCols:_col3, iceberg_bucket(_col2, 3) + Select Operator [SEL_51] (rows=1 width=#Masked#) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] + Filter Operator [FIL_47] (rows=1 width=#Masked#) + predicate:(_col23 is null and _col46 is null and _col35 is null) + Please refer to the previous Select Operator [SEL_44] + Reducer 4 vectorized + File Output Operator [FS_65] table:{"name:":"default.merge_tmp_table"} - Select Operator [SEL_60] (rows=1 width=#Masked#) + Select Operator [SEL_64] (rows=1 width=#Masked#) Output:["_col0"] - Filter Operator [FIL_59] (rows=1 width=#Masked#) + Filter Operator [FIL_63] (rows=1 width=#Masked#) predicate:(_col4 > 1L) - Group By Operator [GBY_58] (rows=1 width=#Masked#) + Group By Operator [GBY_62] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_55] + SHUFFLE [RS_57] PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_54] (rows=1 width=#Masked#) + Group By Operator [GBY_56] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col39, _col6, _col44 - Select Operator [SEL_50] (rows=1 width=#Masked#) + Select Operator [SEL_52] (rows=1 width=#Masked#) Output:["_col4","_col6","_col39","_col44"] - Filter Operator [FIL_46] (rows=1 width=#Masked#) + Filter Operator [FIL_48] (rows=1 width=#Masked#) predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) - Please refer to the previous Select Operator [SEL_42] + Please refer to the previous Select Operator [SEL_44] Stage-7 Stats Work{} Stage-3 diff --git a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out index 7ba771285ca4..b052deb10003 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out @@ -316,21 +316,21 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: (c = 66) (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (c = 66) (type: boolean) - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), b (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col8 - Statistics: Num rows: 1 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -339,10 +339,10 @@ STAGE PLANS: Select Operator expressions: 33 (type: int), _col8 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -451,20 +451,20 @@ STAGE PLANS: alias: ice01.branch_test1 filterExpr: a is not null (type: boolean) Snapshot ref: branch_test1 - Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: a is not null (type: boolean) - Statistics: Num rows: 4 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 @@ -476,21 +476,21 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col4 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 10 Data size: 2465 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 10 Data size: 2465 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 397 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 301 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -498,14 +498,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -513,14 +513,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) - Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 482 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 482 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -528,14 +528,14 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: _col9 is null (type: boolean) - Statistics: Num rows: 6 Data size: 1479 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -543,24 +543,24 @@ STAGE PLANS: name: default.ice01 Filter Operator predicate: (_col9 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) outputColumnNames: _col2, _col4, _col5, _col6 - Statistics: Num rows: 5 Data size: 1383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) - minReductionHashAggr: 0.4 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) null sort order: zzzz sort order: ++++ Map-reduce partition columns: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE value expressions: _col4 (type: bigint) Reducer 3 Execution mode: vectorized @@ -570,17 +570,17 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col4 > 1L) (type: boolean) - Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cardinality_violation(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index efdbebcf9b54..ff85a29bd341 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -419,6 +419,7 @@ iceberg.llap.query.files=\ vectorized_iceberg_read_parquet.q iceberg.llap.query.compactor.files=\ + iceberg_compaction_colstats_compute.q,\ iceberg_major_compaction_partition_evolution.q,\ iceberg_major_compaction_partition_evolution2.q,\ iceberg_major_compaction_partition_evolution_ordered.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 379f89dc3d4f..b0786216df44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -357,6 +357,17 @@ default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table tab return false; } + /** + * Check if the storage handler can set col statistics of the given granularity. A handler keeps + * them either per partition or for the table as a whole, and one that keeps any says which. + * @param table table object + * @param partitionLevel whether the statistics asked about are the per partition ones + * @return true if the storage handler can set col statistics of that granularity + */ + default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, boolean partitionLevel) { + return false; + } + /** * Returns the row count of the table, letting queries like count(1) be answered from statistics. * @param hmsTable table object diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index fa244289832f..7c357b680409 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -160,7 +160,7 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met // and then if it is not followed by column stats, we should clean // column stats // FIXME: move this to ColStat related part - if (!work.isExplicitAnalyze() && !followedColStats1) { + if (!work.isExplicitAnalyze() && !followedColStats1 && !rewritesWithoutChangingRows()) { StatsSetupConst.clearColumnStatsState(parameters); } @@ -218,6 +218,15 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met return p.getOutput(); } + /** + * Whether this write rewrites files without changing a row, as a compaction does: what it did + * not gather still describes the table. A compaction that is meant to refresh the statistics + * recomputes them, here or in the worker that ran it. + */ + private boolean rewritesWithoutChangingRows() { + return SessionState.get() != null && SessionState.get().isCompaction(); + } + public void collectFileStatus(Warehouse wh, HiveConf conf) throws MetaException, IOException { this.conf = conf; if (providedBasicStats == null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index dd79cdf95c72..22df96e85ae4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -60,7 +60,6 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -410,6 +409,10 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p inferAndSetPrimaryKey(stats.getNumRows(), colStats); stats.setColumnStatsState(deriveStatType(colStats, neededColumns)); + // they describe the whole table, so a scan pruned to part of it is described in part + if (!partList.getReferredPartCols().isEmpty() || partList.hasUnknownPartitions()) { + stats.updateColumnStatsState(State.PARTIAL); + } stats.addToColumnStats(colStats); if (partStats.isEmpty()) { @@ -2032,11 +2035,10 @@ public static boolean isPartitionStats(Table table, HiveConf conf) { public static boolean isPartitionStatsEnabled(Table table, HiveConf conf) { // the metastore keeps a single row of column statistics per table, with nowhere to put a // partition's, so a table partitioned outside it can only keep them per partition itself - if (table.isNonNative() && !table.getStorageHandler().canSetColStatistics(table)) { - return false; + if (table.isNonNative()) { + return table.getStorageHandler().canSetColStatistics(table, true); } - return conf.getBoolVar(DDLUtils.isIcebergTable(table) ? - ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL : ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS); + return conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS); } public static boolean checkCanProvideStats(Table table) { From 6e297683f8a094d602139873126a829506a6e1aa Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 24 Aug 2026 13:48:07 +0300 Subject: [PATCH 07/15] HIVE-29829: Scope partition ANALYZE by partition identity, and name the null partition __HIVE_DEFAULT_PARTITION__ A partition scoped ANALYZE scoped its scan by a value predicate, which names rows rather than partitions. On a table whose partitioning had changed, that predicate reached rows an older spec still held in a partition of its own, and the group that formed described the whole of that partition by the few rows it saw. A table evolved from unpartitioned answered max(id) as 7 where the truth was 100. The scan is now scoped by partition identity, as compaction scopes its rewrite: the values name partitions of the current spec, and the spec's own transforms applied to those values keep every row of each and still prune what the scan reads. A partition of an older spec keeps the statistics it has, which no write since could have changed, because a write lands in the current spec. Rendering a partition of no value as the text "null" also gave it the name a value of that text takes, so the two shared a name and a statistics blob. Hive names it __HIVE_DEFAULT_PARTITION__ and Iceberg's own Conversions reads that name back as no value, so both now say so, which lets a statement name that partition too. --- .../mr/hive/HiveIcebergOutputCommitter.java | 16 +- .../mr/hive/HiveIcebergStorageHandler.java | 12 +- .../iceberg/mr/hive/IcebergTableUtil.java | 67 ++++---- .../compaction/IcebergCompactionService.java | 2 +- .../compaction/IcebergCompactionUtil.java | 42 ++--- .../compaction/IcebergQueryCompactor.java | 29 ++-- .../evaluator/CompactionEvaluator.java | 10 +- .../mr/hive/stats/ColStatsWritePolicy.java | 2 +- .../mr/hive/TestHiveIcebergStatistics.java | 151 +++++++++++++++++- .../iceberg_colstats_analyze_evolution.q | 23 ++- .../iceberg_colstats_analyze_evolution.q.out | 128 ++++++++++++++- .../iceberg_insert_into_partition.q.out | 40 ++--- .../positive/iceberg_stats_with_ppr.q.out | 6 +- .../iceberg_minor_compaction_bucket.q.out | 2 +- .../compactor/TestIcebergCompactorOnTez.java | 8 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 81 ++++++++-- .../hadoop/hive/ql/parse/TransformSpec.java | 10 +- 17 files changed, 488 insertions(+), 141 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java index 2a8701379086..f66f098654b7 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java @@ -505,9 +505,9 @@ private void commitTable(FileIO io, ExecutorService executor, OutputTable output .orElse(RewritePolicy.DEFAULT.name())); if (rewritePolicy != RewritePolicy.DEFAULT) { - String partitionPath = jobContexts.stream() + String partitionName = jobContexts.stream() .findAny() - .map(x -> x.getJobConf().get(IcebergCompactionService.PARTITION_PATH)) + .map(x -> x.getJobConf().get(IcebergCompactionService.PARTITION_NAME)) .orElse(null); long fileSizeThreshold = jobContexts.stream() @@ -516,7 +516,7 @@ private void commitTable(FileIO io, ExecutorService executor, OutputTable output .map(Long::parseLong) .orElse(-1L); - commitCompaction(table, snapshotId, startTime, filesForCommit, partitionPath, fileSizeThreshold); + commitCompaction(table, snapshotId, startTime, filesForCommit, partitionName, fileSizeThreshold); } else { commitOverwrite(table, branchName, snapshotId, startTime, filesForCommit); } @@ -623,14 +623,14 @@ private void commit(Transaction txn, SnapshotUpdate update) { * @param snapshotId The snapshot id of the table to use for validation * @param startTime The start time of the commit - used only for logging * @param results The object containing the new files - * @param partitionPath The path of the compacted partition + * @param partitionName The path of the compacted partition */ private void commitCompaction(Table table, Long snapshotId, long startTime, FilesForCommit results, - String partitionPath, long fileSizeThreshold) { + String partitionName, long fileSizeThreshold) { List existingDataFiles = - IcebergCompactionUtil.getDataFiles(table, snapshotId, partitionPath, fileSizeThreshold); + IcebergCompactionUtil.getDataFiles(table, snapshotId, partitionName, fileSizeThreshold); List existingDeleteFiles = fileSizeThreshold == -1 ? - IcebergCompactionUtil.getDeleteFiles(table, snapshotId, partitionPath) : Collections.emptyList(); + IcebergCompactionUtil.getDeleteFiles(table, snapshotId, partitionName) : Collections.emptyList(); Transaction txn = IcebergAcidUtil.getOrCreateTransaction(table, jobConf); @@ -644,7 +644,7 @@ private void commitCompaction(Table table, Long snapshotId, long startTime, File } commit(txn, rewriteFiles); LOG.info("Compaction commit took {} ms for table: {} partition: {} with {} file(s)", - System.currentTimeMillis() - startTime, table, StringUtils.defaultString(partitionPath, "N/A"), + System.currentTimeMillis() - startTime, table, StringUtils.defaultString(partitionName, "N/A"), results.dataFiles().size()); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index e7e384ca7389..111024a4c7fa 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -194,7 +194,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.SerializationUtil; import org.apache.iceberg.util.SnapshotUtil; @@ -2189,9 +2188,7 @@ public void validatePartSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Objects.requireNonNull(field, String.format("%s is not a partition column", spec.getKey())); // If the partition spec value is null, it's a dynamic partition column. if (spec.getValue() != null) { - Object partKeyVal = Conversions.fromPartitionString(field.type(), spec.getValue()); - Objects.requireNonNull(partKeyVal, - String.format("Partition spec value for column : %s is invalid", field.name())); + IcebergTableUtil.parsePartitionValue(field.type(), spec.getValue()); } } } @@ -2429,7 +2426,6 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta } Set partitions = Sets.newHashSet(); - String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULT_PARTITION_NAME); try (CloseableIterable tasks = scan.planFiles()) { FluentIterable.from(tasks) @@ -2439,10 +2435,8 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta PartitionData partitionData = IcebergTableUtil.toPartitionData(task.partition(), spec.partitionType()); String partName = IcebergTableUtil.toPartitionName(spec, partitionData); - Map partSpecMap = - IcebergTableUtil.makeSpecFromName(partName, spec, partitionData, defaultPartitionName); - - DummyPartition partition = new DummyPartition(hmsTable, partName, partSpecMap); + DummyPartition partition = + new DummyPartition(hmsTable, partName, IcebergTableUtil.specFromName(partName)); partitions.add(partition); }); } catch (IOException e) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index e69b15967a04..9f903308f126 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -547,23 +547,6 @@ public static void performMetadataDelete(Table icebergTable, String branchName, deleteFiles.deleteFromRowFilter(exp).commit(); } - /** - * Parses an Iceberg partition path into a Hive-compatible spec map, representing null partition - * values with the Hive default partition name. - */ - public static Map makeSpecFromName(String partName, PartitionSpec spec, PartitionData data, - String defaultPartitionName) { - Map partSpecMap = Maps.newLinkedHashMap(); - Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); - - List fields = spec.fields(); - for (int i = 0; i < fields.size(); i++) { - if (data.get(i) == null) { - partSpecMap.put(fields.get(i).name(), defaultPartitionName); - } - } - return partSpecMap; - } public static PartitionData toPartitionData(StructLike key, Types.StructType keyType) { PartitionData keyTemplate = new PartitionData(keyType); @@ -582,10 +565,27 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy * name an unpartitioned spec renders. Statistics and partition pruning join on this name, so both must * render it the same way. */ + @SuppressWarnings("unchecked") public static String toPartitionName(PartitionSpec spec, StructLike data) { String path = spec.partitionToPath(data); // an unpartitioned spec renders nothing: its rows belong to the table-level partition - return path.isEmpty() ? DummyPartition.VOID : path; + if (path.isEmpty()) { + return DummyPartition.VOID; + } + // a field of no value renders as the text "null", which a value of that text renders as too, + // so the partition of no value takes the name Hive keeps for it and the two stay two + List fields = spec.fields(); + Class[] javaClasses = spec.javaClasses(); + String[] parts = path.split("/"); + + boolean renamed = false; + for (int i = 0; i < fields.size() && i < parts.length; i++) { + if (data.get(i, (Class) javaClasses[i]) == null) { + parts[i] = fields.get(i).name() + "=" + ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal; + renamed = true; + } + } + return renamed ? String.join("/", parts) : path; } /** @@ -697,6 +697,13 @@ private static Expression buildExpression(Map partitionSpec, return predicate; } + static Object parsePartitionValue(Type type, String value) { + boolean isNullValue = NULL_VALUE.equalsIgnoreCase(value) && + !Types.StringType.get().equals(type); + return isNullValue ? + null : Conversions.fromPartitionString(type, value); + } + private static Expression buildFieldPredicate(Types.StructType partitionType, String fieldName, String value, String keyPrefix) { String key = keyPrefix + fieldName; @@ -747,13 +754,6 @@ private static Expression buildTransformedFieldPredicate( return Expressions.equal(partitionKey, transformedValue); } - private static Object parsePartitionValue(Type type, String value) { - boolean isNullValue = NULL_VALUE.equalsIgnoreCase(value) && - !Types.StringType.get().equals(type); - return isNullValue ? - null : Conversions.fromPartitionString(type, value); - } - /** * Returns a partition matching the given partition spec. * With partition evolution, multiple partitions may match; returns the one from the highest spec ID. @@ -1329,7 +1329,8 @@ public static > Set getPartitionNames(Table ice int tableSpecId = icebergTable.spec().specId(); for (T file : files) { if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { - String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); + String partName = toPartitionName( + icebergTable.specs().get(file.specId()), file.partition()); partitions.add(partName); } } @@ -1340,13 +1341,21 @@ public static List convertNameToMetastorePartition(org.apache.hadoop. Collection partNames) { List partitions = Lists.newArrayList(); for (String partName : partNames) { - Map partSpecMap = Maps.newLinkedHashMap(); - Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); - partitions.add(new DummyPartition(hmsTable, partName, partSpecMap)); + partitions.add(new DummyPartition(hmsTable, partName, specFromName(partName))); } return partitions; } + /** + * The values a partition name encodes. The name is one this table rendered, so it always parses, + * which Warehouse.makeSpecFromName(String) does not get to assume. + */ + static Map specFromName(String partName) { + Map partSpecMap = Maps.newLinkedHashMap(); + Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); + return partSpecMap; + } + public static TableFetcher getTableFetcher(IMetaStoreClient msc, String catalogName, String dbPattern, String tablePattern) { return new TableFetcher.Builder(msc, catalogName, dbPattern, tablePattern).tableTypes( diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java index 4fababd707e4..20649f580681 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; public class IcebergCompactionService extends CompactionService { - public static final String PARTITION_PATH = "compaction_partition_path"; + public static final String PARTITION_NAME = "compaction_partition_name"; private static final String CLASS_NAME = IcebergCompactionService.class.getName(); private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java index 46bf6269f759..fad31bc9ed4d 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionUtil.java @@ -29,6 +29,7 @@ import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.hive.IcebergTableUtil; import org.apache.iceberg.relocated.com.google.common.collect.Lists; public class IcebergCompactionUtil { @@ -40,43 +41,44 @@ private IcebergCompactionUtil() { /** * This method implements a common filter that is used in several places in Iceberg compaction code. * Its aim is to determine if the provided file needs to be handled when compacting a partition whose path is equal to - * the provided partitionPath. Returns true when one of the following conditions is true, otherwise returns false: + * the provided partitionName. Returns true when one of the following conditions is true, otherwise returns false: * 1. table is unpartitioned - * 2. partitionPath is null and the file belongs to the non-latest partition spec - * 3. partitionPath is not null and the file belongs to the partition whose path is the partitionPath + * 2. partitionName is null and the file belongs to the non-latest partition spec + * 3. partitionName is not null and the file belongs to the partition of that name * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name * @param file Data or Delete file */ - public static boolean shouldIncludeForCompaction(Table table, String partitionPath, ContentFile file) { + public static boolean shouldIncludeForCompaction(Table table, String partitionName, ContentFile file) { return !table.spec().isPartitioned() || - partitionPath == null && file.specId() != table.spec().specId() || - partitionPath != null && - table.specs().get(file.specId()).partitionToPath(file.partition()).equals(partitionPath); + partitionName == null && file.specId() != table.spec().specId() || + partitionName != null && + IcebergTableUtil.toPartitionName(table.specs().get(file.specId()), file.partition()) + .equals(partitionName); } - public static boolean shouldIncludeForCompaction(Table table, String partitionPath, ContentFile file, + public static boolean shouldIncludeForCompaction(Table table, String partitionName, ContentFile file, long fileSizeThreshold) { - return shouldIncludeForCompaction(table, partitionPath, file) && + return shouldIncludeForCompaction(table, partitionName, file) && (fileSizeThreshold == -1 || file.fileSizeInBytes() < fileSizeThreshold); } /** * Returns table's list of data files as following: * 1. If the table is unpartitioned, returns all data files. - * 2. If partitionPath is not provided, returns all data files that belong to the non-latest partition spec. - * 3. If partitionPath is provided, returns all data files that belong to the corresponding partition. + * 2. If partitionName is not provided, returns all data files that belong to the non-latest partition spec. + * 3. If partitionName is provided, returns all data files that belong to the corresponding partition. * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name */ - public static List getDataFiles(Table table, Long snapshotId, String partitionPath, + public static List getDataFiles(Table table, Long snapshotId, String partitionName, long fileSizeThreshold) { CloseableIterable scanTasks = table.newBatchScan().useSnapshot(snapshotId).planFiles(); CloseableIterable filteredScanTasks = CloseableIterable.filter(scanTasks, t -> { DataFile file = t.asFileScanTask().file(); - return shouldIncludeForCompaction(table, partitionPath, file, fileSizeThreshold); + return shouldIncludeForCompaction(table, partitionName, file, fileSizeThreshold); }); return Lists.newArrayList(CloseableIterable.transform(filteredScanTasks, t -> t.asFileScanTask().file())); } @@ -84,19 +86,19 @@ public static List getDataFiles(Table table, Long snapshotId, String p /** * Returns table's list of delete files as following: * 1. If the table is unpartitioned, returns all delete files. - * 2. If partitionPath is not provided, returns all delete files that belong to the non-latest partition spec. - * 3. If partitionPath is provided, returns all delete files that belong to corresponding partition. + * 2. If partitionName is not provided, returns all delete files that belong to the non-latest partition spec. + * 3. If partitionName is provided, returns all delete files that belong to corresponding partition. * @param table the iceberg table - * @param partitionPath partition path + * @param partitionName partition name */ - public static List getDeleteFiles(Table table, Long snapshotId, String partitionPath) { + public static List getDeleteFiles(Table table, Long snapshotId, String partitionName) { Table deletesTable = MetadataTableUtils.createMetadataTableInstance(table, MetadataTableType.POSITION_DELETES); CloseableIterable deletesScanTasks = deletesTable.newBatchScan().useSnapshot(snapshotId).planFiles(); CloseableIterable filteredDeletesScanTasks = CloseableIterable.filter(deletesScanTasks, t -> { DeleteFile file = ((PositionDeletesScanTask) t).file(); - return shouldIncludeForCompaction(table, partitionPath, file); + return shouldIncludeForCompaction(table, partitionName, file); }); return Lists.newArrayList(CloseableIterable.transform(filteredDeletesScanTasks, t -> ((PositionDeletesScanTask) t).file())); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java index f2514b70b1f1..5e3f4960eec1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java @@ -21,10 +21,8 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.Warehouse; @@ -45,6 +43,7 @@ import org.apache.hadoop.hive.ql.txn.compactor.CompactorUtil; import org.apache.hadoop.hive.ql.txn.compactor.QueryCompactor; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hive.iceberg.org.apache.orc.storage.common.TableName; import org.apache.iceberg.PartitionField; @@ -194,7 +193,7 @@ private String buildPartitionCompactionQuery( HiveConf.setBoolVar(conf, ConfVars.HIVE_CONVERT_JOIN, false); conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false); HiveConf.setVar(conf, ConfVars.REWRITE_POLICY, RewritePolicy.PARTITION.name()); - conf.set(IcebergCompactionService.PARTITION_PATH, new Path(ci.partName).toString()); + conf.set(IcebergCompactionService.PARTITION_NAME, ci.partName); PartitionSpec spec; String partitionPredicate; @@ -228,20 +227,22 @@ private String buildPartitionPredicate(CompactionInfo ci, PartitionSpec spec) th Types.StructType partitionType = spec.partitionType(); return partitionType.fields().stream().map(field -> { + String column = HiveUtils.unparseIdentifier(field.name()); String value = partSpecMap.get(field.name()); - String literal = "NULL"; - - if (value != null && !value.equals("null")) { - String type = HiveSchemaUtil.convertToTypeString(field.type()); - PartitionField partitionField = partitionFieldMap.get(field.name()); - TransformSpec transformSpec = TransformSpec.fromString(partitionField.transform().toString(), field.name()); - literal = TypeInfoUtils.convertStringToLiteralForSQL( - HiveIcebergFilterFactory.convertPartitionLiteral(value, transformSpec).toString(), - ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(type)).getPrimitiveCategory()); + + if (value == null || value.equals(ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal)) { + return String.format("`partition`.%s IS NULL", column); } - return String.format("`partition`.%s %s %s", HiveUtils.unparseIdentifier(field.name()), - Objects.equals(literal, "NULL") ? "IS" : "=", literal); + TransformSpec transformSpec = TransformSpec.fromString( + partitionFieldMap.get(field.name()).transform().toString(), field.name()); + TypeInfo type = TypeInfoUtils.getTypeInfoFromTypeString(HiveSchemaUtil.convertToTypeString(field.type())); + + String literal = TypeInfoUtils.convertStringToLiteralForSQL( + HiveIcebergFilterFactory.convertPartitionLiteral(value, transformSpec).toString(), + ((PrimitiveTypeInfo) type).getPrimitiveCategory()); + + return String.format("`partition`.%s = %s", column, literal); }).collect(Collectors.joining(" AND ")); } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java index e3f177a842c8..e3b64c39e191 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java @@ -185,9 +185,9 @@ public static double getDeleteFileRatio(Map parameters) { .orElse(TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT); } - private static Pair getPartitionSpecStructPair(Table table, String partitionPath) + private static Pair getPartitionSpecStructPair(Table table, String partitionName) throws IOException { - if (!table.spec().isPartitioned() || partitionPath == null) { + if (!table.spec().isPartitioned() || partitionName == null) { return null; } PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils @@ -200,10 +200,10 @@ private static Pair getPartitionSpecStructPair(Table table, PartitionSpec spec = table.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); PartitionData partitionData = IcebergTableUtil.toPartitionData(data, Partitioning.partitionType(table), spec.partitionType()); - String path = spec.partitionToPath(partitionData); - return Maps.immutableEntry(path, Pair.of(spec.specId(), data)); + String partName = IcebergTableUtil.toPartitionName(spec, partitionData); + return Maps.immutableEntry(partName, Pair.of(spec.specId(), data)); }) - .filter(e -> e.getKey().equals(partitionPath)) + .filter(e -> e.getKey().equals(partitionName)) .transform(Map.Entry::getValue) .get(0); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java index 65b67346888f..9ffbe71535d5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java @@ -211,6 +211,6 @@ private static boolean isFullTableRewrite(Configuration conf) { /** Whether the compaction was pointed at one partition: a spec-evolution one carries PARTITION too. */ private static boolean isSinglePartitionRewrite(Configuration conf) { - return conf.get(IcebergCompactionService.PARTITION_PATH) != null; + return conf.get(IcebergCompactionService.PARTITION_NAME) != null; } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 0320adca9bb2..e28bb764c09c 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -83,6 +83,9 @@ */ public class TestHiveIcebergStatistics extends HiveIcebergStorageHandlerWithEngineBase { + /** What a partition holding no value for a partition column is named. */ + private static final String NULL_PART = "__HIVE_DEFAULT_PARTITION__"; + @Parameterized.Parameter(4) public String statsSource; @@ -417,6 +420,137 @@ public void testStatsWithPartitionedInsertOverwrite() { Assert.assertEquals(7L, colStatsObj(colStats, "p=b", "id").getStatsData().getLongStats().getLowValue()); } + @Test + public void testPartitionScopedAnalyzeLeavesThePartitionItCannotNameAlone() { + // the rows a table held before it was partitioned belong to a partition no value names, so a + // partition scoped ANALYZE reaches some of them and must not describe that partition by those + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_void"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (100, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertEquals(100L, colStatsObj(readCurrentColStats(identifier), DummyPartition.VOID, "id") + .getStatsData().getLongStats().getHighValue()); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals("the partition the statement cannot name keeps describing all of its rows", + 100L, colStatsObj(colStats, DummyPartition.VOID, "id") + .getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("and the one it named describes only the rows that partition holds", + 5L, colStatsObj(colStats, "p=a", "id").getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeCanNameThePartitionOfNoValue() { + // the name Hive gives that partition is the one a statement gives back to reach it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_named_null"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, p1 string, p2 string) PARTITIONED BY SPEC (p1, p2) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, 'a', 'x'), (4, 'a', NULL), (7, 'a', 'null')"); + + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p1='a', p2='" + + NULL_PART + "') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + Assert.assertEquals("only the partition holding no value for p2 was measured", + Set.of("p1=a/p2=" + NULL_PART), colStatsPartitions(testTables.loadTable(identifier))); + // the row holding the text "null" belongs to a partition of its own, which this never named + ColumnStatisticsObj id = colStatsObj(colStats, "p1=a/p2=" + NULL_PART, "id"); + Assert.assertEquals(4L, id.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(4L, id.getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresEveryPartitionThePartialSpecNames() { + // naming some of the partition columns names every partition that agrees on them, and the + // statistics of each stand for that partition alone + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_keys"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, p1 string, p2 string) PARTITIONED BY SPEC (p1, p2) " + + "STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + + " VALUES (1, 'a', 'x'), (9, 'a', 'y'), (7, 'b', 'x'), (4, 'a', NULL)"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p1='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + // the partition holding no value for p2 is named apart from one holding the text of one + Assert.assertEquals("every partition agreeing on p1 was measured, and nothing else", + Set.of("p1=a/p2=x", "p1=a/p2=y", "p1=a/p2=" + NULL_PART), + colStatsPartitions(testTables.loadTable(identifier))); + Assert.assertEquals(1L, colStatsObj(colStats, "p1=a/p2=x", "id") + .getStatsData().getLongStats().getHighValue()); + Assert.assertEquals(9L, colStatsObj(colStats, "p1=a/p2=y", "id") + .getStatsData().getLongStats().getHighValue()); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresATransformPartitionWhole() { + // a value names rows, and the partition holding them holds more; naming it measures all of it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_trunc"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (truncate(1, p)) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'aa'), (9, 'ab'), (7, 'bb')"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='aa') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj truncatedToA = colStatsObj(colStats, "p_trunc=a", "id"); + Assert.assertEquals("the rows the value named are not the only ones the partition holds", + 1L, truncatedToA.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(9L, truncatedToA.getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("the partition the statement never named stays undescribed", + Set.of("p_trunc=a"), colStatsPartitions(testTables.loadTable(identifier))); + } + + @Test + public void testPartitionScopedAnalyzeMeasuresTheCurrentSpecPartitionAlone() { + // a write lands in the current spec, so the partitions of an older one keep describing + // themselves, and naming a partition measures the one the table writes today + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_evo_multispec"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (2, 'a'), (7, 'b')"); + shell.executeStatement("ALTER TABLE " + identifier + " SET PARTITION SPEC (p, truncate(1, p))"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'a')"); + + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List colStats = readCurrentColStats(identifier); + ColumnStatisticsObj currentSpec = colStatsObj(colStats, "p=a/p_trunc_1=a", "id"); + Assert.assertEquals(9L, currentSpec.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(9L, currentSpec.getStatsData().getLongStats().getHighValue()); + Assert.assertEquals("the older spec's partitions are not the ones the statement named", + Set.of("p=a/p_trunc_1=a"), colStatsPartitions(testTables.loadTable(identifier))); + } + @Test public void testPartitionScopedAnalyzeWithoutStoredColStats() { // there is nothing stored to carry, so the partition it named is all the file has to hold @@ -562,7 +696,8 @@ public void testStatsWithNullPartitionValue() { " VALUES (1, null), (7, timestamp'2024-06-15 12:00:00')"); List colStats = readCurrentColStats(identifier); - Assert.assertEquals(1L, colStatsObj(colStats, "ts_day=null", "id").getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(1L, colStatsObj(colStats, "ts_day=" + NULL_PART, "id") + .getStatsData().getLongStats().getLowValue()); Assert.assertEquals(7L, colStatsObj(colStats, "ts_day=2024-06-15", "id").getStatsData().getLongStats().getLowValue()); } @@ -1037,7 +1172,7 @@ public void testGetAggrColStatsForPartitioned() throws Exception { @Test public void testGetAggrColStatsForNullAndEmptyPartitions() throws Exception { - // NULL and empty-string partition values render as "last_name=null" and "last_name=" on both the + // NULL and empty-string partition values render apart, and do so on both the // blob-write side (ANALYZE) and the pruned-name side; a rendering mismatch silently drops the // partition from the aggregation and partial aggregation extrapolates fabricated NDVs assumeParquetHiveCatalogIceberg(); @@ -1052,7 +1187,7 @@ public void testGetAggrColStatsForNullAndEmptyPartitions() throws Exception { List partNames = partitionNames(handler, hmsTable); Assert.assertEquals(partNames.toString(), 5, partNames.size()); - Assert.assertTrue(partNames.toString(), partNames.contains("last_name=null")); + Assert.assertTrue(partNames.toString(), partNames.contains("last_name=" + NULL_PART)); Assert.assertTrue(partNames.toString(), partNames.contains("last_name=")); // the blobs must carry the read side's names: an empty-string value that decodes as null would @@ -1248,7 +1383,7 @@ public void testAggrColStatsAfterEvolutionFromUnpartitioned() throws Exception { HiveIcebergStorageHandler handler = storageHandler(); List statNames = Lists.newArrayList(partitionNames(handler, hmsTable)); statNames.add(DummyPartition.VOID); - Assert.assertTrue(statNames.toString(), statNames.contains("last_name=null")); + Assert.assertTrue(statNames.toString(), statNames.contains("last_name=" + NULL_PART)); // customer ids 0..2 exist only among the legacy unpartitioned rows, 3..6 in the partitioned ones assertAggrColStatsRange(identifier, "customer_id", statNames, 0, 6); @@ -2093,7 +2228,7 @@ public void testVoidTransformEvolutionUnifiesPartitionNames() { Assert.assertTrue(hasColStatsForCurrentSnapshot(identifier)); List colStats = readCurrentColStats(identifier); - ColumnStatisticsObj id = colStatsObj(colStats, "a=x/b=null", "id"); + ColumnStatisticsObj id = colStatsObj(colStats, "a=x/b=" + NULL_PART, "id"); Assert.assertEquals(1L, id.getStatsData().getLongStats().getLowValue()); Assert.assertEquals(2L, id.getStatsData().getLongStats().getHighValue()); } @@ -2121,10 +2256,10 @@ public void testAggrColStatsAfterBucketAndYearEvolutionsFromUnpartitioned() thro // the null source values produce null partition values; the empty string hashes to a genuine bucket List statNames = Stream.of( DummyPartition.VOID, - "a_bucket_8=" + bucket(8, "x"), "a_bucket_8=" + bucket(8, ""), "a_bucket_8=null", + "a_bucket_8=" + bucket(8, "x"), "a_bucket_8=" + bucket(8, ""), "a_bucket_8=" + NULL_PART, "a_bucket_4=" + bucket(4, "x") + "/b_year=2023", "a_bucket_4=" + bucket(4, "") + "/b_year=2024", - "a_bucket_4=null/b_year=null") + "a_bucket_4=" + NULL_PART + "/b_year=" + NULL_PART) .sorted().toList(); Assert.assertEquals(statNames, colStatsPartNames(identifier)); @@ -2236,7 +2371,7 @@ public void testPartitionNameRendersAcrossEvolutionsAndTypes() { Assert.assertEquals(DummyPartition.VOID, rows.get(0)[1]); Assert.assertEquals("s=a+b", rows.get(1)[1]); Assert.assertEquals("ts_day=2024-06-01", rows.get(2)[1]); - Assert.assertEquals("ts_day=null", rows.get(3)[1]); + Assert.assertEquals("ts_day=" + NULL_PART, rows.get(3)[1]); Integer bucket = Transforms.bucket(4).bind(Types.StringType.get()).apply("bucketed"); Assert.assertEquals("s_bucket_4=" + bucket, rows.get(4)[1]); Assert.assertEquals("s_trunc_2=tr/ts_month=2025-02", rows.get(5)[1]); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q index dd16808061a1..66b258bafde7 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_colstats_analyze_evolution.q @@ -22,17 +22,21 @@ analyze table ice_evo compute statistics for columns; explain select id from ice_evo where id > 0; +select min(id), max(id) from ice_evo; + -- a write reaching one partition leaves the others describing themselves insert into ice_evo values (7, 'a'); explain select id from ice_evo where id > 0; +select min(id), max(id) from ice_evo; + -- naming that partition measures it again; the ones it never named are carried analyze table ice_evo partition (p='a') compute statistics for columns; explain select id from ice_evo where id > 0; -select count(*) from ice_evo; +select min(id), max(id) from ice_evo; drop table ice_evo; @@ -47,14 +51,31 @@ analyze table ice_evo2 compute statistics for columns; explain select id from ice_evo2 where id > 0; +select min(id), max(id) from ice_evo2; + alter table ice_evo2 set partition spec (p, truncate(1, p)); insert into ice_evo2 values (9, 'a'); explain select id from ice_evo2 where id > 0; +select min(id), max(id) from ice_evo2; + +-- what the user sees before reaching for a partition scoped ANALYZE: partitions of both specs +show partitions ice_evo2; + +-- naming a partition measures the one the current spec writes today; the older spec's p=a keeps +-- the statistics it already had, which no write since could have changed +analyze table ice_evo2 partition (p='a') compute statistics for columns; + +explain select id from ice_evo2 where id > 0; + +select min(id), max(id) from ice_evo2; + analyze table ice_evo2 compute statistics for columns; explain select id from ice_evo2 where id > 0; +select min(id), max(id) from ice_evo2; + drop table ice_evo2; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out index fdcb07b04a1c..38460ab27d23 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_analyze_evolution.q.out @@ -90,6 +90,15 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: select min(id), max(id) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 100 PREHOOK: query: insert into ice_evo values (7, 'a') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table @@ -143,6 +152,15 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: select min(id), max(id) from ice_evo +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 100 PREHOOK: query: analyze table ice_evo partition (p='a') compute statistics for columns PREHOOK: type: ANALYZE_TABLE PREHOOK: Input: default@ice_evo @@ -202,15 +220,15 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select count(*) from ice_evo +PREHOOK: query: select min(id), max(id) from ice_evo PREHOOK: type: QUERY PREHOOK: Input: default@ice_evo PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from ice_evo +POSTHOOK: query: select min(id), max(id) from ice_evo POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_evo POSTHOOK: Output: hdfs://### HDFS PATH ### -4 +1 100 PREHOOK: query: drop table ice_evo PREHOOK: type: DROPTABLE PREHOOK: Input: default@ice_evo @@ -300,6 +318,15 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 7 PREHOOK: query: alter table ice_evo2 set partition spec (p, truncate(1, p)) PREHOOK: type: ALTERTABLE_SETPARTSPEC PREHOOK: Input: default@ice_evo2 @@ -360,6 +387,92 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 +PREHOOK: query: show partitions ice_evo2 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@ice_evo2 +POSTHOOK: query: show partitions ice_evo2 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@ice_evo2 +p=a +p=a/p_trunc_1=a +p=b +PREHOOK: query: analyze table ice_evo2 partition (p='a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: default@ice_evo2 +PREHOOK: Output: default@ice_evo2@p=a +PREHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_evo2 partition (p='a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2 +POSTHOOK: Output: default@ice_evo2@p=a +POSTHOOK: Output: default@ice_evo2@p=a/p_trunc_1=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain select id from ice_evo2 where id > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select id from ice_evo2 where id > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_evo2 + filterExpr: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (id > 0L) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 PREHOOK: query: analyze table ice_evo2 compute statistics for columns PREHOOK: type: ANALYZE_TABLE PREHOOK: Input: default@ice_evo2 @@ -421,6 +534,15 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: select min(id), max(id) from ice_evo2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_evo2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(id), max(id) from ice_evo2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_evo2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 9 PREHOOK: query: drop table ice_evo2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@ice_evo2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index 1793758f29f3..e17468137436 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -267,14 +267,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 3 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -283,17 +283,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -303,14 +303,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -374,14 +374,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -390,17 +390,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -410,14 +410,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out index 57bdbac57c57..c5146d8505e9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats_with_ppr.q.out @@ -54,14 +54,14 @@ STAGE PLANS: TableScan alias: ice01 filterExpr: ((day = 3) and (year = 2023) and (month = 10)) (type: boolean) - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 3bb172c0df60..987c779d87fb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -171,7 +171,7 @@ CompactionId Database Table Partition Type State Worker host Worker Enqueue Time #Masked# default srcbucket_big key_bucket_8=0 MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big key_bucket_8=3 MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big key_bucket_8=4 MINOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default srcbucket_big key_bucket_8=null MINOR succeeded #Masked# manual default 0 0 0 --- +#Masked# default srcbucket_big key_bucket_8=__HIVE_DEFAULT_PARTITION__ MINOR succeeded #Masked# manual default 0 0 0 --- #Masked# default srcbucket_big --- MINOR succeeded #Masked# manual default 0 0 0 --- PREHOOK: query: desc formatted default.srcbucket_big PREHOOK: type: DESCTABLE diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java index 5e22dde0d97a..c73a90c26805 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java @@ -71,7 +71,11 @@ public void testIcebergCompactorWithAllPartitionFieldTypes() throws Exception{ CompactorTestUtil.runCompaction(conf, DB_NAME, TABLE_NAME, CompactionType.MINOR, false, "a=aaa111/a_trunc=aaa/a_bucket=0/b=1/c=100/d=1.0/e=2.0/f=4.00/g=true/h=2024-05-01/h_year=2024/i_month=2024-05/j_day=2024-05-01/k=2024-05-02T10%3A00%3A00/k_hour=2024-05-02-10", "a=bbb222/a_trunc=bbb/a_bucket=3/b=2/c=200/d=2.0/e=3.0/f=8.00/g=false/h=2024-05-03/h_year=2024/i_month=2024-05/j_day=2024-05-03/k=2024-05-04T13%3A00%3A00/k_hour=2024-05-04-13", - "a=null/a_trunc=null/a_bucket=null/b=null/c=null/d=null/e=null/f=null/g=null/h=null/h_year=null/i_month=null/j_day=null/k=null/k_hour=null" + "a=__HIVE_DEFAULT_PARTITION__/a_trunc=__HIVE_DEFAULT_PARTITION__/a_bucket=__HIVE_DEFAULT_PARTITION__/" + + "b=__HIVE_DEFAULT_PARTITION__/c=__HIVE_DEFAULT_PARTITION__/d=__HIVE_DEFAULT_PARTITION__/" + + "e=__HIVE_DEFAULT_PARTITION__/f=__HIVE_DEFAULT_PARTITION__/g=__HIVE_DEFAULT_PARTITION__/" + + "h=__HIVE_DEFAULT_PARTITION__/h_year=__HIVE_DEFAULT_PARTITION__/i_month=__HIVE_DEFAULT_PARTITION__/" + + "j_day=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__/k_hour=__HIVE_DEFAULT_PARTITION__" ); Assert.assertEquals(3, getFilesCount()); @@ -118,7 +122,7 @@ public void testIcebergAutoCompactionPartitionEvolution() throws Exception { // Compaction should be initiated for each partition from the latest spec Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=aaa", CompactionType.MINOR, CompactionState.SUCCEEDED)); Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=bbb", CompactionType.MINOR, CompactionState.SUCCEEDED)); - Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=null", CompactionType.MINOR, CompactionState.SUCCEEDED)); + Assert.assertTrue(isCompactExist(rsp, "b_trunc_3=__HIVE_DEFAULT_PARTITION__", CompactionType.MINOR, CompactionState.SUCCEEDED)); // Additional compaction should be initiated for all partitions from past partition specs Assert.assertTrue(isCompactExist(rsp, null, CompactionType.MINOR, CompactionState.SUCCEEDED)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 2a785d4af081..20b48f40208f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; @@ -50,6 +52,7 @@ import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsField; import org.apache.hadoop.hive.ql.stats.ColStatsProcessor.ColumnStatsType; import org.apache.hadoop.hive.ql.stats.StatsUtils; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -209,6 +212,56 @@ private static CharSequence genPartitionClause(Table tbl, Map pa return predPresent ? whereClause.append(groupByClause) : groupByClause; } + /** + * Narrows the scan to the partitions of the current spec the statement named, which it has. A value + * predicate names rows, not partitions, so on its own it reaches into a partition of any spec + * holding a row that carries the value, and a group formed of those would describe that partition + * by part of what it holds. The partitions of one spec share its transforms, so applying them to + * the value named holds every row of each partition named, and prunes what the scan reads. + */ + private static CharSequence genNamedPartitionClause(Table tbl, Map named, HiveConf conf) + throws SemanticException { + List partitions = tbl.getStorageHandler().getPartitions(tbl, named, true); + if (partitions.isEmpty()) { + throw new SemanticException( + ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() + " : " + named); + } + String scanned = partitions.stream() + .map(Partition::getName) + .map(name -> genPartValueString(serdeConstants.STRING_TYPE_NAME, name)) + .collect(Collectors.joining(", ", + " where " + unparseIdentifier(PARTITION_NAME.getName(), conf) + " in (", ")")); + String pruned = partitionValuePredicate(tbl, named, conf); + return pruned.isEmpty() ? scanned : scanned + " and " + pruned; + } + + /** The partition columns the statement gave a value, which are the ones it named. */ + private static Map namedPartitionValues(Map partSpec) { + return partSpec == null ? Collections.emptyMap() : + partSpec.entrySet().stream() + .filter(part -> part.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * What the named partitions hold, as their spec decides it: the transform of a partition field + * takes the same value for every row of the partition, so comparing it against the transform of + * the value named keeps all of them. + */ + private static String partitionValuePredicate(Table tbl, Map named, HiveConf conf) { + return tbl.getStorageHandler().getPartitionTransformSpec(tbl).stream() + .filter(spec -> named.containsKey(spec.getColumnName())) + .map(spec -> { + String value = named.get(spec.getColumnName()); + // the partition of no value is named, not valued, and a transform of nothing is nothing + return ConfVars.DEFAULT_PARTITION_NAME.defaultStrVal.equals(value) ? + spec.toHiveExpr(conf) + " is null" : + spec.toHiveExpr(conf) + " = " + spec.toHiveExpr( + genPartValueString(getColTypeOf(tbl, spec.getColumnName()), value)); + }) + .collect(Collectors.joining(" and ")); + } + private static String getColTypeOf(Table tbl, String partKey) { for (FieldSchema fs : tbl.getPartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { @@ -262,7 +315,8 @@ protected static List getFieldSchemasByColName(Table tbl, List partTransformSpec, Map partSpec, boolean isPartitionStats) { + List partTransformSpec, Map partSpec, boolean isPartitionStats) + throws SemanticException { String rewritten = genRewrittenQuery(tbl, columnSchemas, conf, partTransformSpec, partSpec, isPartitionStats, false); isRewritten = true; @@ -274,14 +328,15 @@ private String genRewrittenQuery(FieldSchemas columnSchemas, HiveConf conf, * included in the input table. */ protected static String genRewrittenQuery(Table tbl, HiveConf conf, - List partTransformSpec, Map partSpec, boolean isPartitionStats) { + List partTransformSpec, Map partSpec, boolean isPartitionStats) + throws SemanticException { return ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, getStatsEligibleFieldSchemas(tbl), conf, partTransformSpec, partSpec, isPartitionStats, true); } private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, HiveConf conf, List partTransformSpec, Map partSpec, boolean isPartitionStats, - boolean useTableValues) { + boolean useTableValues) throws SemanticException { StringBuilder rewrittenQueryBuilder = new StringBuilder("select "); StringBuilder columnNamesBuilder = new StringBuilder(); @@ -297,11 +352,7 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, H final String columnName = unparseIdentifier(columnSchema.getName(), conf); final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(columnSchema.getType()); - try { - genComputeStats(rewrittenQueryBuilder, conf, i, columnName, typeInfo); - } catch (SemanticException e) { - throw new RuntimeException(e); - } + genComputeStats(rewrittenQueryBuilder, conf, i, columnName, typeInfo); columnNamesBuilder.append(columnName); @@ -360,10 +411,14 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, H } else if (!tbl.hasNonNativePartitionSupport()) { rewrittenQueryBuilder.append(genPartitionClause(tbl, partSpec, conf)); } else { - // the name groups rows of every spec alike; the predicate keeps the scan to the partitions - // the statement named, which only the current spec's own columns can express - rewrittenQueryBuilder.append(genPartitionPredicate(tbl, partSpec, conf)) - .append("\ngroup by ").append(unparseIdentifier(PARTITION_NAME.getName(), conf)); + // the name groups rows of every spec alike, so the scan is held to the partitions the + // statement named by naming them too + Map named = namedPartitionValues(partSpec); + if (!named.isEmpty()) { + rewrittenQueryBuilder.append(genNamedPartitionClause(tbl, named, conf)); + } + rewrittenQueryBuilder.append("\ngroup by ") + .append(unparseIdentifier(PARTITION_NAME.getName(), conf)); } } @@ -536,7 +591,7 @@ private static void appendKllSketch(StringBuilder rewrittenQueryBuilder, HiveCon private static void appendBitVector(StringBuilder rewrittenQueryBuilder, HiveConf conf, String columnName) throws SemanticException { - String func = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_NDV_ALGO).toLowerCase(); + String func = HiveConf.getVar(conf, ConfVars.HIVE_STATS_NDV_ALGO).toLowerCase(); if ("hll".equals(func)) { rewrittenQueryBuilder .append("compute_bit_vector_hll(") diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java index a205f4099c77..ee7e39efa50b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TransformSpec.java @@ -121,11 +121,15 @@ public static String toSourceStruct(List partTransformSpec, Confi } public String toHiveExpr(Configuration conf) { - String identifier = unparseIdentifier(columnName, conf); + return toHiveExpr(unparseIdentifier(columnName, conf)); + } + + /** The transform applied to an operand, which is a column of its own table or a value of one. */ + public String toHiveExpr(String operand) { if (transformType == TransformSpec.TransformType.IDENTITY) { - return identifier; + return operand; } - String fn = "iceberg_" + transformType.name().toLowerCase() + "(" + identifier; + String fn = "iceberg_" + transformType.name().toLowerCase() + "(" + operand; switch (transformType) { case BUCKET: case TRUNCATE: From a1f26c50e177f6d0caeaf4dfa0ef97e714841896 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Wed, 26 Aug 2026 01:08:43 +0300 Subject: [PATCH 08/15] HIVE-29829: Stream column statistics through a pull-based writer, framed for column pruning The gather held every partition's statistics in memory before handing them over in one call, because a Puffin file cannot be appended once closed. It can be streamed while open: setColStatistics now takes an iterator, the handler pulls batch by batch, and each partition's blob is written as it arrives, so the whole of a large table's statistics is never held at once. One table commit still publishes the file. A partition blob frames one zstd-compressed slice per column behind a header naming each slice's position, so a read fetches and deserializes only the columns it was asked for: a whole read for broad asks and small blobs, positioned coalesced reads for narrow asks on big ones. Sliced blobs are read through one stream over one InputFile built with the known file size, so a wide unpruned aggregate costs one open instead of one per blob. A merge carries the entries of unmeasured, unchanged partitions bytes for bytes, without deserializing them. Every entry travels under one versioned blob type; released table-level files keep serving, and files of the older partition layout read as absent rather than wrong. The format lives in IcebergColStatsWriter and IcebergColStatsReader beside the policy, which now nests in the writer as WritePolicy over the facts of one Write; a plain insert may no longer stand in for partitions it only added to, which the compile-time gather gate already guaranteed from the other side. A pull that failed after the first batch was caught by the writer and reported as an unsuccessful write, so the statement finished with its statistics silently dropped, which hive.stats.reliable forbids. The writer now catches only its own write failures; anything else, the gather above all, propagates and fails the statement, with the temporary Puffin file cleaned up either way. A table-level merge completed every computed column, so a column absent from the stored file - measured by no ANALYZE - entered with the increment's rows standing for the whole. It now completes only the columns the stored file already describes; the rest wait for an ANALYZE. Compaction matched a request's partition against Iceberg's raw path, which renders the partition of no value differently than the name Hive gives it, so that partition's compaction crashed choosing its spec and would have selected no files: both matches now speak toPartitionName. Two goldens stale since before this change are regenerated: a partition-column scan under table-level statistics reads PARTIAL, and statistics after unmeasured merge-on-read writes read NONE rather than the frozen values. --- .../mr/hive/HiveIcebergOutputCommitter.java | 2 +- .../mr/hive/HiveIcebergStorageHandler.java | 110 +--- .../iceberg/mr/hive/IcebergTableUtil.java | 219 +------- .../compaction/IcebergCompactionService.java | 27 +- .../mr/hive/stats/ColStatsWritePolicy.java | 216 -------- .../mr/hive/stats/IcebergColStatsReader.java | 272 ++++++++++ .../mr/hive/stats/IcebergColStatsWriter.java | 507 ++++++++++++++++++ .../mr/hive/TestHiveIcebergStatistics.java | 103 +++- .../hive/stats/TestIcebergColStatsFormat.java | 113 ++++ ...va => TestIcebergColStatsWritePolicy.java} | 22 +- .../iceberg_insert_overwrite_partition.q.out | 38 +- ...major_compaction_partition_evolution.q.out | 50 +- .../hive/ql/metadata/HiveStorageHandler.java | 15 +- .../hive/ql/stats/BasicStatsNoJobTask.java | 4 +- .../hadoop/hive/ql/stats/BasicStatsTask.java | 4 +- .../hive/ql/stats/ColStatsProcessor.java | 62 ++- 16 files changed, 1147 insertions(+), 617 deletions(-) delete mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java create mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java create mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java create mode 100644 iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java rename iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/{TestColStatsWritePolicy.java => TestIcebergColStatsWritePolicy.java} (88%) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java index f66f098654b7..67e6199e28ac 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java @@ -623,7 +623,7 @@ private void commit(Transaction txn, SnapshotUpdate update) { * @param snapshotId The snapshot id of the table to use for validation * @param startTime The start time of the commit - used only for logging * @param results The object containing the new files - * @param partitionName The path of the compacted partition + * @param partitionName The name of the compacted partition */ private void commitCompaction(Table table, Long snapshotId, long startTime, FilesForCommit results, String partitionName, long fileSizeThreshold) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 111024a4c7fa..5032037f7783 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -60,10 +61,10 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -182,7 +183,8 @@ import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.hive.actions.HiveIcebergDeleteOrphanFiles; import org.apache.iceberg.mr.hive.plan.IcebergBucketFunction; -import org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsReader; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter; import org.apache.iceberg.mr.hive.udf.GenericUDFIcebergZorder; import org.apache.iceberg.puffin.BlobMetadata; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; @@ -436,7 +438,6 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese return predicate; } - @Override public boolean canProvideBasicStatistics() { return true; @@ -723,29 +724,17 @@ public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsT } @Override - public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colStats) { + public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, + Iterator colStats) { Table tbl = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); // a write to a branch moves that branch's head, leaving the table's current snapshot behind Snapshot snapshot = IcebergTableUtil.getTableSnapshot(tbl, hmsTable); - if (snapshot == null || colStats.isEmpty()) { + if (snapshot == null || !colStats.hasNext()) { return false; } - ColStatsWritePolicy policy = ColStatsWritePolicy.resolve(tbl, snapshot, colStats, conf); - if (policy == ColStatsWritePolicy.SKIP) { - return IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf); - } - try { - if (policy == ColStatsWritePolicy.MERGE && !checkAndMergeColStats(colStats, tbl, snapshot)) { - return false; - } - } catch (Exception e) { - LOG.warn("Unable to merge column stats: {}", e.getMessage()); - return false; - } - return IcebergTableUtil.writeColStats(tbl, snapshot, colStats, conf); + return IcebergColStatsWriter.writeColStats(tbl, snapshot, colStats, conf); } - @Override public boolean canProvideColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); @@ -789,7 +778,7 @@ public List getColStatistics(org.apache.hadoop.hive.ql.meta filter = null; } - return IcebergTableUtil.readColStats(table, snapshot.snapshotId(), filter); + return IcebergColStatsReader.readColStats(table, snapshot.snapshotId(), filter); } @Override @@ -815,18 +804,19 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms // a partition written since the file was written is no longer described by it Predicate upToDate = IcebergTableUtil.upToDateColStats(table, snapshot, statsFile, conf, true); - Predicate filter = metadata -> { - String partition = metadata.properties().get(PARTITION); - return partitions.contains(partition) && upToDate.test(partition); - }; - - List partStats = IcebergTableUtil.readColStats(table, snapshot.snapshotId(), true, filter); - - Set asked = Sets.newHashSet(colNames); - partStats.forEach(colStats -> - colStats.getStatsObj().removeIf(statsObj -> !asked.contains(statsObj.getColName()))); - // the metastore counts a partition as found only when it has every column asked about - partStats.removeIf(colStats -> colStats.getStatsObj().size() != colNames.size()); + Map> statsByPart = IcebergColStatsReader.readPartColStats(table, statsFile, + partition -> partitions.contains(partition) && upToDate.test(partition), Sets.newHashSet(colNames)); + + List partStats = Lists.newArrayList(); + statsByPart.forEach((partition, statsObjs) -> { + // the metastore counts a partition as found only when it has every column asked about + if (statsObjs.size() == colNames.size()) { + ColumnStatisticsDesc statsDesc = + new ColumnStatisticsDesc(false, hmsTable.getDbName(), hmsTable.getTableName()); + statsDesc.setPartName(partition); + partStats.add(new ColumnStatistics(statsDesc, statsObjs)); + } + }); List colStatsList = MetaStoreServerUtils.aggrPartitionStats(partStats, MetaStoreUtils.getDefaultCatalog(conf), hmsTable.getDbName(), hmsTable.getTableName(), @@ -894,59 +884,6 @@ private String getStatsSource() { .toUpperCase(); } - - /** - * Completes the statistics a write computed with the ones already stored, for the two writes that - * describe less than the whole table: an INSERT, whose table-level statistics cover the rows it - * added, and an INSERT OVERWRITE or ANALYZE of some partitions, which leaves the rest untouched. - *

- * Which stored statistics may be drawn on differs with that. An INSERT sums its rows into them, so - * they have to account for every other row the table holds; statistics an unrecorded write has - * already outlived would lose that write for good, so the increment is dropped and only a complete - * rewrite can restart the chain. Partitions are copied rather than summed, so the untouched ones - * may come from the nearest ancestor, which is what a read of them returns anyway. - * - * @return whether the write may proceed - */ - private boolean checkAndMergeColStats(List statsNew, Table tbl, Snapshot snapshot) - throws InvalidObjectException, IOException { - boolean isTblLevel = statsNew.getFirst().getStatsDesc().isIsTblLevel(); - Long parentId = snapshot.parentId(); - // A write commits a snapshot of its own, so what it completes sits on the one before it. An - // ANALYZE commits none: it writes to the snapshot it read, where the statistics already are. - StatisticsFile statsOldSrc = isTblLevel ? - (parentId == null ? null : IcebergTableUtil.getColStatsFile(tbl, parentId, conf)) : - IcebergTableUtil.findColStatsFile(tbl, snapshot.snapshotId(), conf); - if (statsOldSrc == null) { - // a partition describes itself, so with nothing stored there is nothing to carry and what - // was computed stands on its own; a table-level increment has nothing to add itself to - return !isTblLevel; - } - if (!isTblLevel) { - // The file holds one blob per partition. This write replaced the rows of the partitions it - // reached, so its statistics stand for those, and the partitions it never saw carry over. - List statsOld = IcebergTableUtil.readColStatsOrThrow(tbl, statsOldSrc, null); - // two blobs of one partition would leave nothing to merge into, so the later one answers - Map storedByPart = statsOld.stream().collect( - Collectors.toMap(stats -> stats.getStatsDesc().getPartName(), Function.identity(), (a, b) -> b)); - statsNew.forEach(stats -> storedByPart.remove(stats.getStatsDesc().getPartName())); - // Carry what no write since has changed. This is the only place that can decide it without - // a reader walking the history itself, and an entry that no longer describes its partition - // is of no use to any read. - Predicate stillHolds = - IcebergTableUtil.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); - storedByPart.keySet().removeIf(part -> !stillHolds.test(part)); - statsNew.addAll(storedByPart.values()); - return true; - } - List statsOld = IcebergTableUtil.readColStatsOrThrow(tbl, statsOldSrc, null); - ColumnStatistics statsObjOld = new ColumnStatistics(null, statsOld); - if (statsObjOld.getStatsObjSize() != 0 && !statsNew.getFirst().getStatsObj().isEmpty()) { - MetaStoreServerUtils.mergeColStats(statsNew.getFirst(), statsObjOld); - } - return true; - } - /** * Iceberg's optimistic concurrency control fails to provide means for IOW and Insert operations isolation. * Use `hive.txn.ext.locking.enabled` config to create Hive locks in order to guarantee data consistency. @@ -992,7 +929,6 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m .collect(Collectors.toList()); } - @Override public Function partitionNameResolver( org.apache.hadoop.hive.ql.metadata.Table hmsTable, StructObjectInspector inspector) { @@ -1229,7 +1165,6 @@ public void storageHandlerCommit(Properties commitProperties, Operation operatio } } - @Override public HiveIcebergOutputCommitter getOutputCommitter() { return new HiveIcebergOutputCommitter(); @@ -2506,7 +2441,6 @@ public boolean supportsDefaultColumnValues(Map tblProps) { return IcebergTableUtil.formatVersion(tblProps) >= 3; } - private static List schema(List exprs) { return exprs.stream().map(v -> new FieldSchema(v.getName(), v.getTypeInfo().getTypeName(), "")) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 9f903308f126..3315d4080514 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -20,21 +20,17 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.io.Serializable; import java.io.UncheckedIOException; -import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.TreeSet; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; @@ -42,21 +38,16 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; -import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.TableFetcher; @@ -83,8 +74,6 @@ import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.GenericBlobMetadata; -import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.ManifestFiles; @@ -124,15 +113,9 @@ import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.hive.serde.objectinspector.IcebergObjectInspector; -import org.apache.iceberg.puffin.Blob; -import org.apache.iceberg.puffin.BlobMetadata; -import org.apache.iceberg.puffin.Puffin; -import org.apache.iceberg.puffin.PuffinCompressionCodec; -import org.apache.iceberg.puffin.PuffinReader; -import org.apache.iceberg.puffin.PuffinWriter; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -141,8 +124,6 @@ import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.ByteBuffers; -import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PartitionUtil; import org.apache.iceberg.util.SnapshotUtil; import org.apache.iceberg.util.StructProjection; @@ -165,9 +146,8 @@ public class IcebergTableUtil { private static final String SPEC_ID_FIELD = "spec_id"; private static final String NULL_VALUE = "null"; - static final String PARTITION_FIELD = "partition"; + public static final String PARTITION_FIELD = "partition"; private static final String MODIFIED_PARTITIONS_PREFIX = "modifiedPartitions."; - static final String STATS = "/stats/snap-"; private IcebergTableUtil() { } @@ -280,16 +260,8 @@ static Snapshot getTableSnapshot(Table table, String snapshotRef) { return table.currentSnapshot(); } - /** - * The newest column statistics file describing the snapshot: its own, or - across commits that - * rewrite files without changing any rows (compaction) - an ancestor's. Of the granularity the - * table maintains, which is the one every write produces. - */ - static StatisticsFile getColStatsFile(Table table, long snapshotId, Configuration conf) { - return getColStatsFile(table, snapshotId, isPartitionStats(table, conf)); - } - static StatisticsFile getColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + public static StatisticsFile getColStatsFile(Table table, long snapshotId, boolean partitionLevel) { // A rewrite leaves every row in the partition it was already in, and so separates statistics // from nothing - unless the table has evolved, where compaction selects the rows of the older // specs and writes them under the current one. That is the same condition the compactor @@ -309,7 +281,7 @@ static StatisticsFile findColStatsFile(Table table, long snapshotId, Configurati return findColStatsFile(table, snapshotId, isPartitionStats(table, conf)); } - static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { + public static StatisticsFile findColStatsFile(Table table, long snapshotId, boolean partitionLevel) { // a snapshot holding no rows (truncate) ends the walk: what precedes it didn't survive return colStatsFileOf( table, snapshotId, partitionLevel, IcebergTableUtil::isEmptySnapshot); @@ -341,9 +313,12 @@ private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boole private static StatisticsFile lookupColStatsFile(Table table, long snapshotId, boolean partitionLevel) { return table.statisticsFiles().stream() .filter(stats -> stats.snapshotId() == snapshotId) - .filter(stats -> stats.blobMetadata().stream().anyMatch( - metadata -> ColumnStatisticsObj.class.getSimpleName().equals(metadata.type()) && - metadata.properties().containsKey(PARTITION_FIELD) == partitionLevel)) + .filter(stats -> stats.blobMetadata().stream().anyMatch(partitionLevel ? + metadata -> IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + metadata.properties().containsKey(PARTITION_FIELD) : + metadata -> (IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) || + IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(metadata.type())) && + !metadata.properties().containsKey(PARTITION_FIELD))) .findAny().orElse(null); } @@ -547,7 +522,6 @@ public static void performMetadataDelete(Table icebergTable, String branchName, deleteFiles.deleteFromRowFilter(exp).commit(); } - public static PartitionData toPartitionData(StructLike key, Types.StructType keyType) { PartitionData keyTemplate = new PartitionData(keyType); return keyTemplate.copyFor(key); @@ -926,116 +900,6 @@ public static TransformSpec getTransformSpec(Table table, String transformName, return spec; } - /** - * Writes the column statistics as the table's statistics file for the current snapshot: one blob - * per column at table level, one blob per partition otherwise, each carrying the partition name - * the read side joins on. Everything written describes the snapshot it is written for, so a read - * asks only what happened after it. - */ - @SuppressWarnings("checkstyle:CyclomaticComplexity") - static boolean writeColStats(Table tbl, Snapshot snapshot, List colStats, - Configuration conf) { - try { - StatisticsFile statisticsFile; - String statsPath = tbl.location() + STATS + UUID.randomUUID(); - - try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) - .createdBy(Constants.HIVE_ENGINE) - .build()) { - - long snapshotId = snapshot.snapshotId(); - long snapshotSequenceNumber = snapshot.sequenceNumber(); - Schema schema = tbl.spec().schema(); - - boolean first = true; - - for (ColumnStatistics stats : colStats) { - boolean isTblLevel = stats.getStatsDesc().isIsTblLevel(); - - Map properties = isTblLevel ? Map.of() : - Map.of(PARTITION_FIELD, String.valueOf(stats.getStatsDesc().getPartName())); - - List statsObjects = isTblLevel ? - stats.getStatsObj() : List.of(stats); - - // a column dropped or renamed since the entry was stored resolves no field: its - // statistics leave with it - stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); - - List fieldIds = null; - - if (!isTblLevel) { - // For partition-level stats, we emit one blob per partition; - // therefore, only the first blob should contain the actual fieldIds. - fieldIds = !first ? List.of(-1) : - stats.getStatsObj().stream() - .map(obj -> schema.caseInsensitiveFindField(obj.getColName()).fieldId()) - .toList(); - first = false; - } - - for (Serializable statsObj : statsObjects) { - byte[] serialized = SerializationUtils.serialize(statsObj); - - if (isTblLevel) { - fieldIds = List.of(schema.caseInsensitiveFindField( - ((ColumnStatisticsObj) statsObj).getColName()).fieldId()); - } - - writer.add(new Blob( - ColumnStatisticsObj.class.getSimpleName(), - fieldIds, - snapshotId, - snapshotSequenceNumber, - ByteBuffer.wrap(serialized), - PuffinCompressionCodec.NONE, - properties - )); - } - } - - writer.finish(); - - statisticsFile = - new GenericStatisticsFile( - snapshotId, - statsPath, - writer.fileSize(), - writer.footerSize(), - writer.writtenBlobsMetadata().stream() - .map(GenericBlobMetadata::from) - .collect(ImmutableList.toImmutableList()) - ); - } catch (IOException e) { - LOG.warn("Unable to write column stats to the Puffin file: {}", e.getMessage()); - - Path path = new Path(statsPath); - FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - fs.delete(path, false); - } - return false; - } - tbl.updateStatistics() - .setStatistics(statisticsFile) - .commit(); - return true; - - } catch (Exception e) { - LOG.warn("Unable to write column stats: {}", e.getMessage()); - } - return false; - } - - /** - * Whether the stored column statistics still describe the snapshot the table names: its current - * one, or a branch's head, which a write to that branch moves on its own. - */ - public static boolean colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Configuration conf) { - Table table = getTable(conf, hmsTable.getTTable()); - Snapshot snapshot = getTableSnapshot(table, hmsTable); - return snapshot != null && colStatsAccurate(table, snapshot, conf); - } /** * Whether the stored column statistics still describe the table: the current snapshot owns them, @@ -1063,7 +927,9 @@ static Set partitionsChangedSince(Table table, Snapshot snapshot, long s sinceSnapshotId + '.' + capped; Optional cached = SessionStateUtil.getResource(conf, cacheKey); if (cached.isPresent()) { - return ((Optional>) cached.get()).orElse(null); + @SuppressWarnings("unchecked") + Optional> hit = (Optional>) cached.get(); + return hit.orElse(null); } Set changed = walkPartitionsChangedSince(table, snapshot, sinceSnapshotId, conf, capped); SessionStateUtil.addResource(conf, cacheKey, Optional.ofNullable(changed)); @@ -1106,8 +972,7 @@ private static Set walkPartitionsChangedSince(Table table, Snapshot snap /** * Records which partitions a snapshot changed, naming each file under its own spec. False when * a delete of no partition is reached, since it applies to the rows of every one and names none. - * Stored - * entries are named the same way: an ANALYZE names each group after the file its rows came from, + * Stored entries are named the same way: an ANALYZE names each group after the file its rows came from, * and a write only ever lands in a partition of the spec current when it ran. */ private static boolean collectChangedPartitions(Table table, Snapshot snapshot, Set changed) { @@ -1135,67 +1000,13 @@ private static Iterable> changedFiles(Table table, Snap * written for, so only what happened after it matters. False for all of them when the writes in * between cannot be traced. */ - static Predicate upToDateColStats(Table table, Snapshot snapshot, + public static Predicate upToDateColStats(Table table, Snapshot snapshot, StatisticsFile statsFile, Configuration conf, boolean capped) { Set changed = partitionsChangedSince(table, snapshot, statsFile.snapshotId(), conf, capped); return partition -> changed != null && !changed.contains(partition); } - /** The stored statistics describing the whole table. */ - static List readColStats(Table table, long snapshotId, Predicate filter) { - return readColStats(table, snapshotId, false, filter); - } - - /** - * The stored statistics of the granularity the caller serves. Asking for one the table doesn't - * keep yields nothing rather than blobs of the other shape. - */ - static List readColStats(Table table, long snapshotId, boolean partitionLevel, - Predicate filter) { - StatisticsFile statsFile = findColStatsFile(table, snapshotId, partitionLevel); - if (statsFile == null) { - LOG.warn("Column stats file not found for snapshot: {}", snapshotId); - return Lists.newArrayList(); - } - try { - return readColStatsOrThrow(table, statsFile, filter); - } catch (Exception e) { - // serving no stats degrades the planner to estimates - never wrong - LOG.warn("Unable to read column stats: {}", e.getMessage()); - return Lists.newArrayList(); - } - } - - /** - * The strict variant for the merge path: an unreadable statistics file must not be mistaken for - * an absent one, or the increment would be persisted as the complete statistics. - */ - static List readColStatsOrThrow(Table table, StatisticsFile statsFile, Predicate filter) - throws IOException { - List colStats = Lists.newArrayList(); - String statsPath = statsFile.path(); - try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)) - .withFileSize(statsFile.fileSizeInBytes()) - .withFooterSize(statsFile.fileFooterSizeInBytes()) - .build()) { - List blobMetadata = reader.fileMetadata().blobs(); - - if (filter != null) { - blobMetadata = blobMetadata.stream().filter(filter) - .toList(); - } - Iterator it = Iterables.transform(reader.readAll(blobMetadata), Pair::second).iterator(); - LOG.info("Using column stats from: {}", statsPath); - - while (it.hasNext()) { - byte[] byteBuffer = ByteBuffers.toByteArray(it.next()); - colStats.add(SerializationUtils.deserialize(byteBuffer)); - } - } - return colStats; - } - public static ExecutorService newDeleteThreadPool(String completeName, int numThreads) { AtomicInteger deleteThreadsIndex = new AtomicInteger(0); return Executors.newFixedThreadPool(numThreads, runnable -> { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java index 20649f580681..ce2e815b0522 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergCompactionService.java @@ -74,26 +74,19 @@ public Boolean compact(Table table, CompactionInfo ci) throws Exception { ci.runAs = TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf); } - try { - CompactorPipeline compactorPipeline = compactorFactory.getCompactorPipeline(table, conf, ci, msc); - // no follow-up ANALYZE: basic statistics live in the table metadata, column statistics are - // gathered by the compaction query itself, and a compaction commit outdates neither - computeStats = false; + CompactorPipeline compactorPipeline = compactorFactory.getCompactorPipeline(table, conf, ci, msc); + // no follow-up ANALYZE: basic statistics live in the table metadata, column statistics are + // gathered by the compaction query itself, and a compaction commit outdates neither + computeStats = false; - LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + ", id:" + - ci.id + " with compute stats set to " + computeStats); + LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + ", id:" + ci.id); - CompactorContext compactorContext = new CompactorContext(conf, table, ci); - compactorPipeline.execute(compactorContext); + CompactorContext compactorContext = new CompactorContext(conf, table, ci); + compactorPipeline.execute(compactorContext); - LOG.info("Completed " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + - ", marking as compacted."); - msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci)); - - } catch (Throwable e) { - computeStats = false; - throw e; - } + LOG.info("Completed " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + + ", marking as compacted."); + msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci)); return true; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java deleted file mode 100644 index 9ffbe71535d5..000000000000 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/ColStatsWritePolicy.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.hive.stats; - -import java.util.List; -import java.util.function.BooleanSupplier; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.ql.Context.RewritePolicy; -import org.apache.hadoop.hive.ql.parse.ColumnStatsSemanticAnalyzer; -import org.apache.hadoop.hive.ql.plan.HiveOperation; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.session.SessionStateUtil; -import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SnapshotSummary; -import org.apache.iceberg.Table; -import org.apache.iceberg.mr.hive.IcebergTableUtil; -import org.apache.iceberg.mr.hive.compaction.IcebergCompactionService; - -/** - * What a write does to a table's column statistics. Iceberg keeps them in one file that is always - * written whole, so a write replaces that file, merges into it, or leaves it alone. - */ -public enum ColStatsWritePolicy { - /** Write the computed statistics, discarding the stored ones. */ - REPLACE, - /** - * Write the computed statistics completed by the stored ones: a table-level write adds its rows - * to them, a partition-level one stands in for the partitions it reached. - */ - MERGE, - /** Write nothing: leave the stored statistics as they are. */ - SKIP; - - /** - * Everything the decision reads, so that it can be made without a session. {@code statsAccurate} - * is a supplier because answering it walks the table's metadata, and most cases never ask. - */ - record ColStatsWrite( - // what this write computed - boolean tableWideStats, - // the table it computed them for, and what it already holds for this snapshot - boolean keepsStatsPerPartition, - boolean partitioned, - BooleanSupplier statsAccurate, - // the statement that computed them - boolean analyze, - boolean analyzePartition, - boolean compaction, - boolean majorCompaction, - boolean fullTableRewrite, - boolean singlePartitionRewrite, - // what its commit did to the rows - boolean holdsOnlyAddedRows, - boolean emptySnapshot, - boolean wroteNoRows, - boolean replacePartitions) { - - boolean isStatsAccurate() { - return statsAccurate.getAsBoolean(); - } - } - - /** - * What to do with the statistics a write computed. - * - * @param snapshot the snapshot the statistics describe: the table's current one, or a branch head - * @param colStats the computed statistics, one entry for the table or one per partition - */ - public static ColStatsWritePolicy resolve( - Table tbl, Snapshot snapshot, List colStats, Configuration conf) { - - return resolve(new ColStatsWrite( - colStats.getFirst().getStatsDesc().isIsTblLevel(), - IcebergTableUtil.isPartitionStats(tbl, conf), - tbl.spec().isPartitioned(), - () -> IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf), - isAnalyze(conf), - isAnalyzePartition(conf), - isCompaction(), - isMajorCompaction(conf), - isFullTableRewrite(conf), - isSinglePartitionRewrite(conf), - holdsOnlyAddedRows(snapshot), - IcebergTableUtil.isEmptySnapshot(snapshot), - wroteNoRows(conf), - Boolean.parseBoolean( - snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP)))); - } - - /** The decision itself, over the facts alone, so that every case of it can be stated as one. */ - static ColStatsWritePolicy resolve(ColStatsWrite write) { - if (!write.tableWideStats()) { - return resolveForPartitions(write); - } - if (write.keepsStatsPerPartition()) { - // the table stores its statistics per partition; these describe it as a whole - return SKIP; - } - if (write.compaction()) { - // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. - return write.fullTableRewrite() && write.majorCompaction() && !write.isStatsAccurate() ? - REPLACE : SKIP; - } - // ANALYZE reads the whole table, so it replaces. - return write.analyze() ? REPLACE : resolveForWrite(write); - } - - /** What a statement does to the file holding one blob per partition. */ - private static ColStatsWritePolicy resolveForPartitions(ColStatsWrite write) { - if (write.analyze()) { - // what it read is what it named, not what the snapshot it reads happens to hold - return write.analyzePartition() ? MERGE : REPLACE; - } - if (write.compaction()) { - // a partition read whole describes itself; part of one describes none of it - return write.singlePartitionRewrite() && write.majorCompaction() ? MERGE : SKIP; - } - if (write.holdsOnlyAddedRows() || write.emptySnapshot()) { - return REPLACE; - } - return write.wroteNoRows() || write.isStatsAccurate() ? SKIP : MERGE; - } - - /** What an INSERT, INSERT OVERWRITE or CTAS does to the stored statistics. */ - private static ColStatsWritePolicy resolveForWrite(ColStatsWrite write) { - if (write.emptySnapshot()) { - // The table is now empty, so stats of the rows it held must go. Must precede the - // wroteNoRows check, which an emptying write also matches. - return REPLACE; - } - if (write.wroteNoRows() || write.isStatsAccurate()) { - // Nothing to record: the statement wrote no rows, or this snapshot already has stats. - return SKIP; - } - if (write.holdsOnlyAddedRows()) { - // Every row came from this write: a CTAS, a whole-table INSERT OVERWRITE, or the first - // INSERT after a TRUNCATE. - return REPLACE; - } - if (write.replacePartitions()) { - // Iceberg flags whole-table and partition overwrites alike, so the partition spec decides: - // only an unpartitioned table had every row replaced. - return write.partitioned() ? SKIP : REPLACE; - } - // An INSERT: its stats cover the rows it added, the stored ones cover the rest. - return MERGE; - } - - /** - * Whether the table holds nothing but the rows this snapshot added. Iceberg carries the row - * total across commits, so it stays above the added count while older rows remain. False if - * either count is missing from the summary. - */ - private static boolean holdsOnlyAddedRows(Snapshot snapshot) { - String added = snapshot.summary().get(SnapshotSummary.ADDED_RECORDS_PROP); - return added != null && added.equals(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP)); - } - - /** Whether the statement wrote no rows, as its file sink reported to the query state. */ - private static boolean wroteNoRows(Configuration conf) { - return SessionStateUtil.getQueryState(conf) - .map(qs -> qs.getNumModifiedRows() == 0) - .orElse(false); - } - - private static boolean isAnalyze(Configuration conf) { - return SessionStateUtil.getQueryState(conf) - .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) - .orElse(false); - } - - /** Whether the ANALYZE named the partitions it is for, leaving the rest of the table alone. */ - private static boolean isAnalyzePartition(Configuration conf) { - return SessionStateUtil.getResource(conf, ColumnStatsSemanticAnalyzer.ANALYZE_PARTITION).isPresent(); - } - - private static boolean isCompaction() { - return SessionState.get() != null && SessionState.get().isCompaction(); - } - - /** Whether the compaction read every file of what it was pointed at: a minor one skips by size. */ - private static boolean isMajorCompaction(Configuration conf) { - return conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; - } - - /** Whether what it was pointed at was the whole table, which only an unpartitioned one is. */ - private static boolean isFullTableRewrite(Configuration conf) { - return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)); - } - - /** Whether the compaction was pointed at one partition: a spec-evolution one carries PARTITION too. */ - private static boolean isSinglePartitionRewrite(Configuration conf) { - return conf.get(IcebergCompactionService.PARTITION_NAME) != null; - } -} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java new file mode 100644 index 000000000000..471df0bd610c --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import io.airlift.compress.zstd.ZstdDecompressor; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.IOUtil; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.mr.hive.IcebergTableUtil; +import org.apache.iceberg.puffin.BlobMetadata; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Reads the column statistics {@link IcebergColStatsWriter} stores: table-level entries one blob + * per column, partition entries one framed blob per partition, fetching and decoding only the + * columns asked. + */ +public final class IcebergColStatsReader { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergColStatsReader.class); + + /** Below this, fetching the blob whole costs less than a second positioned read. */ + static final long WHOLE_BLOB_READ_BYTES = 32 * 1024; + private static final long HEADER_PROBE_BYTES = 4 * 1024; + /** Asked slices closer than this are fetched in one read; a seek costs more than the gap. */ + private static final long COALESCE_GAP_BYTES = 64 * 1024; + + private IcebergColStatsReader() { + } + + /** The stored statistics describing the whole table. */ + public static List readColStats(Table table, long snapshotId, Predicate filter) { + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(table, snapshotId, false); + if (statsFile == null) { + LOG.warn("Column stats file not found for snapshot: {}", snapshotId); + return Lists.newArrayList(); + } + try { + return readColStatsOrThrow(table, statsFile, filter); + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to read column stats: {}", e.getMessage()); + return Lists.newArrayList(); + } + } + + /** + * The strict variant for the merge path: an unreadable statistics file must not be mistaken for + * an absent one, or the increment would be persisted as the complete statistics. + */ + static List readColStatsOrThrow(Table table, StatisticsFile statsFile, + Predicate filter) + throws IOException { + List colStats = Lists.newArrayList(); + String statsPath = statsFile.path(); + try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsPath)) + .withFileSize(statsFile.fileSizeInBytes()) + .withFooterSize(statsFile.fileFooterSizeInBytes()) + .build()) { + List blobMetadata = reader.fileMetadata().blobs(); + + if (filter != null) { + blobMetadata = blobMetadata.stream().filter(filter) + .toList(); + } + Iterator it = Iterables.transform(reader.readAll(blobMetadata), Pair::second).iterator(); + LOG.info("Using column stats from: {}", statsPath); + + while (it.hasNext()) { + byte[] byteBuffer = ByteBuffers.toByteArray(it.next()); + colStats.add(SerializationUtils.deserialize(byteBuffer)); + } + } + return colStats; + } + + /** + * The stored partition entries the given file holds for the partitions the filter admits, each + * trimmed to the asked columns; a null column set asks for all of them. + */ + public static Map> readPartColStats(Table table, StatisticsFile statsFile, + Predicate partitionFilter, Set columns) { + Map> result = Maps.newLinkedHashMap(); + try (PuffinReader reader = Puffin.read(table.io().newInputFile(statsFile.path())) + .withFileSize(statsFile.fileSizeInBytes()) + .withFooterSize(statsFile.fileFooterSizeInBytes()) + .build()) { + List blobs = reader.fileMetadata().blobs().stream() + .filter(metadata -> IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + metadata.properties().containsKey(IcebergTableUtil.PARTITION_FIELD)) + .filter(metadata -> { + String partName = metadata.properties().get(IcebergTableUtil.PARTITION_FIELD); + return partName != null && (partitionFilter == null || partitionFilter.test(partName)); + }) + .toList(); + LOG.info("Using column stats from: {}", statsFile.path()); + List whole = Lists.newArrayList(); + List sliced = Lists.newArrayList(); + + for (BlobMetadata blob : blobs) { + // a narrow ask on a big blob fetches just its own slices; anything else reads the blob whole + if (columns == null || blob.length() <= WHOLE_BLOB_READ_BYTES) { + whole.add(blob); + } else { + sliced.add(blob); + } + } + if (!sliced.isEmpty()) { + // blobs sit in the file in footer order, so one stream serves them all seeking forward + InputFile file = table.io().newInputFile(statsFile.path(), statsFile.fileSizeInBytes()); + try (SeekableInputStream in = file.newStream()) { + for (BlobMetadata blob : sliced) { + result.put(blob.properties().get(IcebergTableUtil.PARTITION_FIELD), + readSlices(in, blob.offset(), blob.length(), columns)); + } + } + } + for (Pair blob : reader.readAll(whole)) { + result.put(blob.first().properties().get(IcebergTableUtil.PARTITION_FIELD), + decodePartitionBlob(blob.second(), columns)); + } + } catch (Exception e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to read column stats: {}", e.getMessage()); + result.clear(); + } + return result; + } + + /** The slices of the asked columns out of a whole blob, the rest skipped without decoding. */ + static List decodePartitionBlob(ByteBuffer blob, Set columns) + throws IOException { + DataInputStream data = new DataInputStream(new ByteArrayInputStream(ByteBuffers.toByteArray(blob))); + if (data.readInt() != IcebergColStatsWriter.PART_STATS_FORMAT_VERSION) { + // a frame this reader does not know reads as absent rather than wrong + return List.of(); + } + int count = data.readInt(); + data.readInt(); // header length: only a positioned read needs it + String[] names = new String[count]; + int[] storedLengths = new int[count]; + int[] rawLengths = new int[count]; + for (int i = 0; i < count; i++) { + names[i] = data.readUTF(); + storedLengths[i] = data.readInt(); + rawLengths[i] = data.readInt(); + } + List statsObjs = Lists.newArrayList(); + for (int i = 0; i < count; i++) { + if (columns == null || columns.contains(names[i])) { + byte[] slice = new byte[storedLengths[i]]; + data.readFully(slice); + statsObjs.add(decodeSlice(slice, rawLengths[i])); + } else { + data.skipNBytes(storedLengths[i]); + } + } + return statsObjs; + } + + /** + * The asked columns of one blob, fetched by position: the header names where each slice sits, so + * only the slices asked for are read at all. Reads coalesce while the gap between asked slices + * costs less than another seek, so projecting 3 of 3000 columns reads about 3 slices. + */ + static List readSlices(SeekableInputStream in, long blobOffset, long blobLength, + Set columns) throws IOException { + byte[] probe = new byte[(int) Math.min(blobLength, HEADER_PROBE_BYTES)]; + in.seek(blobOffset); + IOUtil.readFully(in, probe, 0, probe.length); + DataInputStream head = new DataInputStream(new ByteArrayInputStream(probe)); + if (head.readInt() != IcebergColStatsWriter.PART_STATS_FORMAT_VERSION) { + return List.of(); + } + int count = head.readInt(); + int headerLength = head.readInt(); + if (headerLength > probe.length) { + byte[] header = Arrays.copyOf(probe, headerLength); + IOUtil.readFully(in, header, probe.length, headerLength - probe.length); + head = new DataInputStream(new ByteArrayInputStream(header, 3 * Integer.BYTES, + headerLength - 3 * Integer.BYTES)); + } + String[] names = new String[count]; + int[] storedLengths = new int[count]; + int[] rawLengths = new int[count]; + for (int i = 0; i < count; i++) { + names[i] = head.readUTF(); + storedLengths[i] = head.readInt(); + rawLengths[i] = head.readInt(); + } + long[] offsets = new long[count]; + long offset = headerLength; + for (int i = 0; i < count; i++) { + offsets[i] = offset; + offset += storedLengths[i]; + } + List statsObjs = Lists.newArrayList(); + int cursor = 0; + while (cursor < count) { + if (!columns.contains(names[cursor])) { + cursor++; + continue; + } + int first = cursor; + int last = cursor; + for (int next = cursor + 1; next < count; next++) { + if (columns.contains(names[next])) { + if (offsets[next] - (offsets[last] + storedLengths[last]) > COALESCE_GAP_BYTES) { + break; + } + last = next; + } + } + byte[] range = new byte[(int) (offsets[last] + storedLengths[last] - offsets[first])]; + in.seek(blobOffset + offsets[first]); + IOUtil.readFully(in, range, 0, range.length); + for (int slice = first; slice <= last; slice++) { + if (columns.contains(names[slice])) { + statsObjs.add(decodeSlice(Arrays.copyOfRange(range, (int) (offsets[slice] - offsets[first]), + (int) (offsets[slice] - offsets[first]) + storedLengths[slice]), rawLengths[slice])); + } + } + cursor = last + 1; + } + return statsObjs; + } + + private static ColumnStatisticsObj decodeSlice(byte[] stored, int rawLength) { + byte[] raw = new byte[rawLength]; + new ZstdDecompressor().decompress(stored, 0, stored.length, raw, 0, rawLength); + return SerializationUtils.deserialize(raw); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java new file mode 100644 index 000000000000..4d0da217c6df --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import io.airlift.compress.zstd.ZstdCompressor; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BooleanSupplier; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; +import org.apache.hadoop.hive.ql.Context.RewritePolicy; +import org.apache.hadoop.hive.ql.parse.ColumnStatsSemanticAnalyzer; +import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.hadoop.hive.ql.txn.compactor.CompactorContext; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.mr.hive.IcebergTableUtil; +import org.apache.iceberg.mr.hive.compaction.IcebergCompactionService; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.BlobMetadata; +import org.apache.iceberg.puffin.Puffin; +import org.apache.iceberg.puffin.PuffinCompressionCodec; +import org.apache.iceberg.puffin.PuffinReader; +import org.apache.iceberg.puffin.PuffinWriter; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterators; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes the column statistics of one gather as the table's statistics file, per the policy the + * write's facts resolve to: replacing the stored file, merging into it by carrying what no write + * since has changed, or leaving it alone. The reading side is {@link IcebergColStatsReader}. + * + * At table level the file holds one blob per column. At partition level it holds one blob per + * partition, pulled and written one at a time so the whole of a large table's statistics is never + * held at once. A partition blob frames one slice per column behind a small header, so a read + * deserializes only the columns it was asked for: + * + * content := version, count, header length, count x (column name, stored length, raw length), + * slices + * slice := one column's ColumnStatisticsObj, Java-serialized and zstd-compressed on its own + * + * The blob itself is not compressed, and the header names every slice's position, so a reader can + * fetch the columns it was asked for and no others; the table-level blobs, one small one per + * column, stay compressed whole. + * + * The frame travels under its own blob type, so a file of the older layout reads as absent rather + * than wrong, and a version bump can change the frame without renaming the type. + */ +public final class IcebergColStatsWriter { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergColStatsWriter.class); + + /** + * The blob type of every entry this writer stores. A table-level entry holds one column bare; a + * partition entry carries the partition name as a property and frames its columns, and the frame + * is versioned apart from the name. + */ + public static final String HIVE_COL_STATS_BLOB_V1 = "hive-column-statistics-v1"; + /** What released writers named a table-level entry; read, never written. */ + public static final String LEGACY_COL_STATS_BLOB = ColumnStatisticsObj.class.getSimpleName(); + static final int PART_STATS_FORMAT_VERSION = 1; + private static final String STATS_FILE_PREFIX = "/stats/snap-"; + + private IcebergColStatsWriter() { + } + + /** Everything written describes the snapshot it is written for, so a read asks only what happened after it. */ + public static boolean writeColStats(Table tbl, Snapshot snapshot, Iterator colStats, + Configuration conf) { + ColumnStatistics head = colStats.next(); + WritePolicy policy = WritePolicy.resolve(tbl, snapshot, head, conf); + if (policy == WritePolicy.SKIP) { + return IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf); + } + Iterator all = Iterators.concat(Iterators.singletonIterator(head), colStats); + try { + return head.getStatsDesc().isIsTblLevel() ? + writeTableColStats(tbl, snapshot, all, policy, conf) : + writePartitionColStats(tbl, snapshot, all, policy, conf); + } catch (IOException | InvalidObjectException e) { + // serving no stats degrades the planner to estimates - never wrong + LOG.warn("Unable to write column stats", e); + return false; + } + } + + private static boolean writeTableColStats(Table tbl, Snapshot snapshot, Iterator colStats, + WritePolicy policy, Configuration conf) throws IOException, InvalidObjectException { + // the table's statistics are one entry holding every column: nothing to stream + ColumnStatistics stats = colStats.next(); + if (policy == WritePolicy.MERGE) { + // A write commits a snapshot of its own, so what it completes sits on the one before it. An + // ANALYZE commits none, but replaces rather than merges, so it never asks. + Long parentId = snapshot.parentId(); + StatisticsFile statsOldSrc = parentId == null ? null : + IcebergTableUtil.getColStatsFile(tbl, parentId, false); + if (statsOldSrc == null) { + // a table-level increment has nothing to add itself to + return false; + } + List statsOld = IcebergColStatsReader.readColStatsOrThrow(tbl, statsOldSrc, null); + // drop columns the stored file does not describe: their stats cover only the inserted + // rows, and with nothing to merge into they would stand as stats for the whole table + Set stored = statsOld.stream().map(ColumnStatisticsObj::getColName) + .collect(Collectors.toSet()); + stats.getStatsObj().removeIf(obj -> !stored.contains(obj.getColName())); + if (stats.getStatsObj().isEmpty()) { + return false; + } + MetaStoreServerUtils.mergeColStats(stats, new ColumnStatistics(null, statsOld)); + } + Schema schema = tbl.spec().schema(); + // a column dropped or renamed since the entry was stored resolves no field: its statistics + // leave with it + stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); + return commitColStatsFile(tbl, snapshot, conf, writer -> { + for (ColumnStatisticsObj obj : stats.getStatsObj()) { + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, + List.of(schema.caseInsensitiveFindField(obj.getColName()).fieldId()), + snapshot.snapshotId(), snapshot.sequenceNumber(), + ByteBuffer.wrap(SerializationUtils.serialize(obj)), + PuffinCompressionCodec.ZSTD, + Map.of())); + } + }); + } + + private static boolean writePartitionColStats(Table tbl, Snapshot snapshot, Iterator colStats, + WritePolicy policy, Configuration conf) throws IOException { + Schema schema = tbl.spec().schema(); + Set written = Sets.newHashSet(); + return commitColStatsFile(tbl, snapshot, conf, writer -> { + boolean first = true; + while (colStats.hasNext()) { + ColumnStatistics stats = colStats.next(); + String partName = stats.getStatsDesc().getPartName(); + if (partName == null) { + // a group naming no partition describes none + continue; + } + // a column dropped or renamed since the entry was stored resolves no field: its + // statistics leave with it + stats.getStatsObj().removeIf(obj -> schema.caseInsensitiveFindField(obj.getColName()) == null); + // only the first blob carries the actual fieldIds, so the footer does not repeat them + // once per partition + List fieldIds = first ? stats.getStatsObj().stream() + .map(obj -> schema.caseInsensitiveFindField(obj.getColName()).fieldId()).toList() : + List.of(-1); + first = false; + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, fieldIds, + snapshot.snapshotId(), snapshot.sequenceNumber(), + encodePartitionBlob(stats.getStatsObj()), + PuffinCompressionCodec.NONE, + Map.of(IcebergTableUtil.PARTITION_FIELD, partName))); + written.add(partName); + } + if (policy == WritePolicy.MERGE) { + carryPartitionColStats(tbl, snapshot, writer, written, conf); + } + }); + } + + /** + * Carries forward, bytes for bytes, the stored entries of the partitions this write never + * measured, as long as no write since the stored file changed them. Carrying is the one place + * that can settle that without a reader paying for the walk, so the walk here is uncapped. + */ + private static void carryPartitionColStats(Table tbl, Snapshot snapshot, PuffinWriter writer, + Set written, Configuration conf) throws IOException { + // an ANALYZE commits no snapshot of its own: it writes to the snapshot it read, where the + // statistics already are, so the walk starts there rather than at the parent + StatisticsFile statsOldSrc = IcebergTableUtil.findColStatsFile(tbl, snapshot.snapshotId(), true); + if (statsOldSrc == null) { + // a partition describes itself: with nothing stored there is nothing to carry, and what + // was computed stands on its own + return; + } + Predicate stillHolds = IcebergTableUtil.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); + try (PuffinReader reader = Puffin.read(tbl.io().newInputFile(statsOldSrc.path())) + .withFileSize(statsOldSrc.fileSizeInBytes()) + .withFooterSize(statsOldSrc.fileFooterSizeInBytes()) + .build()) { + List carried = reader.fileMetadata().blobs().stream() + .filter(metadata -> { + String partName = metadata.properties().get(IcebergTableUtil.PARTITION_FIELD); + return HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) && + partName != null && !written.contains(partName) && stillHolds.test(partName); + }) + .toList(); + for (Pair blob : reader.readAll(carried)) { + writer.add(new Blob( + HIVE_COL_STATS_BLOB_V1, List.of(-1), + snapshot.snapshotId(), snapshot.sequenceNumber(), + ByteBuffer.wrap(ByteBuffers.toByteArray(blob.second())), + PuffinCompressionCodec.NONE, + Map.of(IcebergTableUtil.PARTITION_FIELD, + blob.first().properties().get(IcebergTableUtil.PARTITION_FIELD)))); + } + } + } + + @FunctionalInterface + private interface ColStatsBlobWriter { + void write(PuffinWriter writer) throws IOException; + } + + /** Writes one statistics file through the given blobs and commits it for the snapshot. */ + private static boolean commitColStatsFile(Table tbl, Snapshot snapshot, Configuration conf, + ColStatsBlobWriter blobs) throws IOException { + String statsPath = tbl.location() + STATS_FILE_PREFIX + UUID.randomUUID(); + StatisticsFile statisticsFile; + try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) + .createdBy(Constants.HIVE_ENGINE) + .build()) { + blobs.write(writer); + writer.finish(); + statisticsFile = new GenericStatisticsFile( + snapshot.snapshotId(), + statsPath, + writer.fileSize(), + writer.footerSize(), + writer.writtenBlobsMetadata().stream() + .map(GenericBlobMetadata::from) + .collect(ImmutableList.toImmutableList())); + } catch (Exception e) { + Path path = new Path(statsPath); + FileSystem fs = path.getFileSystem(conf); + if (fs.exists(path)) { + fs.delete(path, false); + } + if (!(e instanceof IOException)) { + // not a write failure but the gather itself: the statement must hear about it + throw e; + } + LOG.warn("Unable to write column stats to the Puffin file", e); + return false; + } + tbl.updateStatistics() + .setStatistics(statisticsFile) + .commit(); + return true; + } + + /** One zstd-compressed slice per column behind a header naming each slice's position. */ + static ByteBuffer encodePartitionBlob(List statsObjs) throws IOException { + ZstdCompressor compressor = new ZstdCompressor(); + List slices = Lists.newArrayListWithCapacity(statsObjs.size()); + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + DataOutputStream header = new DataOutputStream(headerBytes); + for (ColumnStatisticsObj obj : statsObjs) { + byte[] raw = SerializationUtils.serialize(obj); + byte[] buffer = new byte[compressor.maxCompressedLength(raw.length)]; + int stored = compressor.compress(raw, 0, raw.length, buffer, 0, buffer.length); + header.writeUTF(obj.getColName()); + header.writeInt(stored); + header.writeInt(raw.length); + slices.add(Arrays.copyOf(buffer, stored)); + } + header.flush(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DataOutputStream data = new DataOutputStream(out); + data.writeInt(PART_STATS_FORMAT_VERSION); + data.writeInt(statsObjs.size()); + // the header's own end, so a reader can fetch it without the slices + data.writeInt(3 * Integer.BYTES + headerBytes.size()); + headerBytes.writeTo(data); + for (byte[] slice : slices) { + data.write(slice); + } + data.flush(); + return ByteBuffer.wrap(out.toByteArray()); + } + + /** + * What a write does to the stored column statistics: replace them, merge into them, or leave + * them alone. They live in one file written whole, so every write is one of these three. + */ + enum WritePolicy { + /** Write the computed statistics, discarding the stored ones. */ + REPLACE, + /** + * Write the computed statistics completed by the stored ones: at table level the write's rows + * add to them, at partition level its partitions replace theirs and the rest carry over. + */ + MERGE, + /** Write nothing: leave the stored statistics as they are. */ + SKIP; + + /** + * Everything the decision reads, so that it can be made without a session. {@code statsAccurate} + * is a supplier because answering it walks the table's metadata, and most cases never ask. + */ + record Facts( + // what this write computed + boolean tableWideStats, + // the table it computed them for, and what it already holds for this snapshot + boolean keepsStatsPerPartition, + boolean partitioned, + BooleanSupplier statsAccurate, + // the statement that computed them + boolean analyze, + boolean analyzePartition, + boolean compaction, + boolean majorCompaction, + boolean fullTableRewrite, + boolean singlePartitionRewrite, + // what its commit did to the rows + boolean holdsOnlyAddedRows, + boolean emptySnapshot, + boolean wroteNoRows, + boolean replacePartitions) { + + boolean isStatsAccurate() { + return statsAccurate.getAsBoolean(); + } + } + + /** + * What to do with the statistics a write computed. + * + * @param snapshot the snapshot the statistics describe: the table's current one, or a branch head + * @param colStats the computed statistics, one entry for the table or one per partition + */ + static WritePolicy resolve( + Table tbl, Snapshot snapshot, ColumnStatistics head, Configuration conf) { + + return resolve(new Facts( + head.getStatsDesc().isIsTblLevel(), + IcebergTableUtil.isPartitionStats(tbl, conf), + tbl.spec().isPartitioned(), + () -> IcebergTableUtil.colStatsAccurate(tbl, snapshot, conf), + isAnalyze(conf), + isAnalyzePartition(conf), + isCompaction(), + isMajorCompaction(conf), + isFullTableRewrite(conf), + isSinglePartitionRewrite(conf), + holdsOnlyAddedRows(snapshot), + IcebergTableUtil.isEmptySnapshot(snapshot), + wroteNoRows(conf), + Boolean.parseBoolean( + snapshot.summary().get(SnapshotSummary.REPLACE_PARTITIONS_PROP)))); + } + + /** The decision itself, over the facts alone, so that every case of it can be stated as one. */ + static WritePolicy resolve(Facts write) { + if (!write.tableWideStats()) { + return resolveForPartitions(write); + } + if (write.keepsStatsPerPartition()) { + // the table stores its statistics per partition; these describe it as a whole + return SKIP; + } + if (write.compaction()) { + // Compaction changes no rows, so only a whole-table one has read enough to refresh stale stats. + return write.fullTableRewrite() && write.majorCompaction() && !write.isStatsAccurate() ? + REPLACE : SKIP; + } + // ANALYZE reads the whole table, so it replaces. + return write.analyze() ? REPLACE : resolveForWrite(write); + } + + /** What a statement does to the file holding one blob per partition. */ + private static WritePolicy resolveForPartitions(Facts write) { + if (write.analyze()) { + // what it read is what it named, not what the snapshot it reads happens to hold + return write.analyzePartition() ? MERGE : REPLACE; + } + if (write.compaction()) { + // a partition read whole describes itself; part of one describes none of it + return write.singlePartitionRewrite() && write.majorCompaction() ? MERGE : SKIP; + } + if (write.holdsOnlyAddedRows() || write.emptySnapshot()) { + return REPLACE; + } + if (write.wroteNoRows() || write.isStatsAccurate()) { + return SKIP; + } + // only a write that replaced its partitions measured everything they now hold; an insert + // measured rows it added to partitions holding more, so it may not stand in for them + return write.replacePartitions() ? MERGE : SKIP; + } + + /** What an INSERT, INSERT OVERWRITE or CTAS does to the stored statistics. */ + private static WritePolicy resolveForWrite(Facts write) { + if (write.emptySnapshot()) { + // The table is now empty, so stats of the rows it held must go. Must precede the + // wroteNoRows check, which an emptying write also matches. + return REPLACE; + } + if (write.wroteNoRows() || write.isStatsAccurate()) { + // Nothing to record: the statement wrote no rows, or this snapshot already has stats. + return SKIP; + } + if (write.holdsOnlyAddedRows()) { + // Every row came from this write: a CTAS, a whole-table INSERT OVERWRITE, or the first + // INSERT after a TRUNCATE. + return REPLACE; + } + if (write.replacePartitions()) { + // Iceberg flags whole-table and partition overwrites alike, so the partition spec decides: + // only an unpartitioned table had every row replaced. + return write.partitioned() ? SKIP : REPLACE; + } + // An INSERT: its stats cover the rows it added, the stored ones cover the rest. + return MERGE; + } + + /** + * Whether the table holds nothing but the rows this snapshot added. Iceberg carries the row + * total across commits, so it stays above the added count while older rows remain. False if + * either count is missing from the summary. + */ + private static boolean holdsOnlyAddedRows(Snapshot snapshot) { + String added = snapshot.summary().get(SnapshotSummary.ADDED_RECORDS_PROP); + return added != null && added.equals(snapshot.summary().get(SnapshotSummary.TOTAL_RECORDS_PROP)); + } + + /** Whether the statement wrote no rows, as its file sink reported to the query state. */ + private static boolean wroteNoRows(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> qs.getNumModifiedRows() == 0) + .orElse(false); + } + + private static boolean isAnalyze(Configuration conf) { + return SessionStateUtil.getQueryState(conf) + .map(qs -> HiveOperation.ANALYZE_TABLE == qs.getHiveOperation()) + .orElse(false); + } + + /** Whether the ANALYZE named the partitions it is for, leaving the rest of the table alone. */ + private static boolean isAnalyzePartition(Configuration conf) { + return SessionStateUtil.getResource(conf, ColumnStatsSemanticAnalyzer.ANALYZE_PARTITION).isPresent(); + } + + private static boolean isCompaction() { + return SessionState.get() != null && SessionState.get().isCompaction(); + } + + /** Whether the compaction read every file of what it was pointed at: a minor one skips by size. */ + private static boolean isMajorCompaction(Configuration conf) { + return conf.get(CompactorContext.COMPACTION_FILE_SIZE_THRESHOLD) == null; + } + + /** Whether what it was pointed at was the whole table, which only an unpartitioned one is. */ + private static boolean isFullTableRewrite(Configuration conf) { + return RewritePolicy.FULL_TABLE.name().equals(HiveConf.getVar(conf, ConfVars.REWRITE_POLICY)); + } + + /** Whether the compaction was pointed at one partition: a spec-evolution one carries PARTITION too. */ + private static boolean isSinglePartitionRewrite(Configuration conf) { + return conf.get(IcebergCompactionService.PARTITION_NAME) != null; + } + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index e28bb764c09c..a1e52181897f 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -25,6 +25,7 @@ import java.io.UncheckedIOException; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -37,6 +38,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.DummyPartition; @@ -57,6 +59,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.hadoop.ConfigProperties; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsReader; import org.apache.iceberg.mr.hive.test.TestTables; import org.apache.iceberg.mr.hive.test.TestTables.TestTableType; import org.apache.iceberg.mr.hive.test.utils.HiveIcebergStorageHandlerTestUtils; @@ -76,7 +79,6 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; - /** * Tests verifying correct statistics generation behaviour on Iceberg tables triggered by: ANALYZE queries, inserts, * CTAS, etc... @@ -285,11 +287,11 @@ public void testBranchWriteStoresItsColStatsOnTheBranch() { Table icebergTable = testTables.loadTable(identifier); long branchSnapshotId = icebergTable.snapshot("b1").snapshotId(); Assert.assertNotNull("the branch's head carries the statistics its write gathered", - IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, partitionLevel(icebergTable))); // the increment extends the fork point's statistics: the table's 0..2 plus the branch's 100 List branchStats = - IcebergTableUtil.readColStats(icebergTable, branchSnapshotId, partitionLevel(icebergTable), null); + IcebergColStatsReader.readColStats(icebergTable, branchSnapshotId, null); ColumnStatisticsObj branchId = branchStats.stream() .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); @@ -323,7 +325,7 @@ public void testAnalyzeOnBranchStoresStatsOnTheBranch() { long branchSnapshotId = testTables.loadTable(identifier).snapshot("b1").snapshotId(); Assert.assertNull("the branch's head starts without statistics", IcebergTableUtil.getColStatsFile( - testTables.loadTable(identifier), branchSnapshotId, shell.getHiveConf())); + testTables.loadTable(identifier), branchSnapshotId, partitionLevel(testTables.loadTable(identifier)))); shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS FOR COLUMNS"); @@ -333,9 +335,9 @@ public void testAnalyzeOnBranchStoresStatsOnTheBranch() { // and it describes the branch's rows, stored on the branch's head Table icebergTable = testTables.loadTable(identifier); Assert.assertNotNull("the branch's head carries the statistics the analyze computed", - IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, shell.getHiveConf())); + IcebergTableUtil.getColStatsFile(icebergTable, branchSnapshotId, partitionLevel(icebergTable))); List branchStats = - IcebergTableUtil.readColStats(icebergTable, branchSnapshotId, partitionLevel(icebergTable), null); + IcebergColStatsReader.readColStats(icebergTable, branchSnapshotId, null); ColumnStatisticsObj branchId = branchStats.stream() .filter(obj -> "customer_id".equals(obj.getColName())).findFirst().orElseThrow(); Assert.assertEquals(0L, branchId.getStatsData().getLongStats().getLowValue()); @@ -1018,7 +1020,7 @@ public void testIcebergColStatsPath() throws IOException { table.refresh(); Path tblColPath = new Path(IcebergTableUtil.getColStatsFile( - table, table.currentSnapshot().snapshotId(), shell.getHiveConf()).path()); + table, table.currentSnapshot().snapshotId(), partitionLevel(table)).path()); Assert.assertNotNull(tblColPath); // Check that if colPath is created correctly Assert.assertTrue(tblColPath.getFileSystem(shell.getHiveConf()).exists(tblColPath)); @@ -1318,8 +1320,7 @@ public void testStatsAfterEvolutionFromUnpartitioned() throws Exception { // blob under the synthetic partition name, so values existing solely among them (Green, Pink) // are accounted there List colStats = - IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); + readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); Assert.assertEquals( List.of(DummyPartition.VOID, "last_name=Barna", "last_name=Brown", "last_name=Rozsaszin", "last_name=Zold"), @@ -1703,7 +1704,6 @@ public void testColStatsServedButFrozenAfterDelete() { Assert.assertEquals(2L, id2023.getStatsData().getLongStats().getHighValue()); } - @Test public void testColStatsNotAccurateAfterExternalWrite() { // an engine that maintains no Hive statistics can commit at any time: the accuracy flag is @@ -1852,6 +1852,60 @@ public void testEmptyWriteKeepsColStats() { Assert.assertTrue(colStatsAccurate(identifier)); } + @Test + public void testMergeCompletesOnlyTheColumnsTheStoredFileDescribes() { + // ANALYZE of one column stores a file describing it alone; the increment an insert gathers + // for the other column has no stored whole to add itself to, so it must not enter as one + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_partial_columns"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + + " (id bigint, v bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 100)"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS id"); + + shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 5)"); + + // the file holds no half-truth for v: the increment's entry was not promoted + List stored = readCurrentColStats(identifier).getFirst().getStatsObj(); + Assert.assertEquals(List.of("id"), stored.stream().map(ColumnStatisticsObj::getColName).toList()); + } + + @Test + public void testGatherFailureFailsTheWriteInsteadOfDroppingIt() { + // a pull that fails after the first batch must fail the statement: reporting success with + // the statistics silently dropped is what hive.stats.reliable forbids + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_failing_gather"); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + ColumnStatistics first = new ColumnStatistics( + new ColumnStatisticsDesc(true, "default", identifier.name()), List.of()); + Iterator failing = new Iterator<>() { + private boolean served = false; + + @Override + public boolean hasNext() { + return true; + } + + @Override + public ColumnStatistics next() { + if (served) { + throw new RuntimeException("Failed to fetch computed column statistics"); + } + served = true; + return first; + } + }; + Assert.assertThrows(RuntimeException.class, () -> storageHandler().setColStatistics(hmsTable, failing)); + Assert.assertFalse("a failed gather must not publish statistics", hasColStatsForCurrentSnapshot(identifier)); + } + @Test public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { // the same insert onto a table that carries no statistics: an increment gathered over no rows @@ -2142,8 +2196,29 @@ private boolean partitionLevel(Table icebergTable) { private List readCurrentColStats(TableIdentifier identifier) { Table icebergTable = testTables.loadTable(identifier); - return IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); + return readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); + } + + /** The stored entries at the table's granularity, one per partition or one for the table. */ + private List readColStats(Table icebergTable, long snapshotId) { + if (!partitionLevel(icebergTable)) { + List statsObjs = IcebergColStatsReader.readColStats(icebergTable, snapshotId, null); + if (statsObjs.isEmpty()) { + return List.of(); + } + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, "default", icebergTable.name()); + return List.of(new ColumnStatistics(statsDesc, statsObjs)); + } + StatisticsFile statsFile = IcebergTableUtil.findColStatsFile(icebergTable, snapshotId, true); + if (statsFile == null) { + return List.of(); + } + return IcebergColStatsReader.readPartColStats(icebergTable, statsFile, null, null).entrySet().stream() + .map(entry -> { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "default", icebergTable.name()); + statsDesc.setPartName(entry.getKey()); + return new ColumnStatistics(statsDesc, entry.getValue()); + }).toList(); } private boolean hasColStatsForCurrentSnapshot(TableIdentifier identifier) { @@ -2208,7 +2283,6 @@ public void testColStatsAfterPartitionTruncate() { Assert.assertEquals(7L, idB.getStatsData().getLongStats().getHighValue()); } - @Test public void testVoidTransformEvolutionUnifiesPartitionNames() { // a V1 removal keeps the field as a void transform: a legacy row with a null value and the @@ -2289,8 +2363,7 @@ private static ColumnStatisticsObj colStatsObj(List colStats, private List colStatsPartNames(TableIdentifier identifier) { Table icebergTable = testTables.loadTable(identifier); List colStats = - IcebergTableUtil.readColStats( - icebergTable, icebergTable.currentSnapshot().snapshotId(), partitionLevel(icebergTable), null); + readColStats(icebergTable, icebergTable.currentSnapshot().snapshotId()); return colStats.stream().map(stats -> stats.getStatsDesc().getPartName()).sorted().toList(); } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java new file mode 100644 index 000000000000..30155afa7bee --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.stats; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.iceberg.Files; +import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.util.ByteBuffers; +import org.junit.Assert; +import org.junit.Test; + +/** The partition blob frame: what a read fetches and decodes is only what it asked for. */ +public class TestIcebergColStatsFormat { + + @Test + public void rangedReadFetchesOnlyTheAskedColumns() throws Exception { + // 3000 columns also push the header past the probe, so it is completed by a second read + List statsObjs = columns(3000); + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(statsObjs); + int blobLength = blob.remaining(); + File file = blobAt(blob, 123); + + List read; + try (SeekableInputStream in = Files.localInput(file).newStream()) { + read = IcebergColStatsReader.readSlices(in, 123, blobLength, Set.of("c0", "c1499", "c2999")); + } + + Assert.assertEquals(List.of("c0", "c1499", "c2999"), + read.stream().map(ColumnStatisticsObj::getColName).toList()); + for (ColumnStatisticsObj obj : read) { + long ordinal = Long.parseLong(obj.getColName().substring(1)); + Assert.assertEquals(ordinal, obj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals(2 * ordinal, obj.getStatsData().getLongStats().getHighValue()); + } + } + + @Test + public void wholeBlobDecodeAgreesWithTheRangedRead() throws Exception { + List statsObjs = columns(40); + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(statsObjs); + int blobLength = blob.remaining(); + File file = blobAt(blob.duplicate(), 0); + + Set asked = Set.of("c7", "c8", "c31"); + Assert.assertEquals( + IcebergColStatsReader.decodePartitionBlob(blob, asked), + readSliced(file, blobLength, asked)); + } + + @Test + public void unknownFrameReadsAsAbsent() throws Exception { + ByteBuffer blob = IcebergColStatsWriter.encodePartitionBlob(columns(3)); + blob.putInt(0, 999); + int blobLength = blob.remaining(); + File file = blobAt(blob.duplicate(), 0); + + Assert.assertEquals(List.of(), + readSliced(file, blobLength, Set.of("c1"))); + Assert.assertEquals(List.of(), IcebergColStatsReader.decodePartitionBlob(blob, Set.of("c1"))); + } + + private static List readSliced(File file, int blobLength, Set asked) + throws Exception { + try (SeekableInputStream in = Files.localInput(file).newStream()) { + return IcebergColStatsReader.readSlices(in, 0, blobLength, asked); + } + } + + private static List columns(int count) { + return IntStream.range(0, count).mapToObj(i -> { + LongColumnStatsData longStats = new LongColumnStatsData(0, i + 1); + longStats.setLowValue(i); + longStats.setHighValue(2L * i); + return new ColumnStatisticsObj("c" + i, "bigint", ColumnStatisticsData.longStats(longStats)); + }).toList(); + } + + /** The blob written at an offset, as it sits inside a Puffin file. */ + private static File blobAt(ByteBuffer blob, int offset) throws Exception { + File file = File.createTempFile("colstats-frame", ".bin"); + file.deleteOnExit(); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(new byte[offset]); + out.write(ByteBuffers.toByteArray(blob)); + } + return file; + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java similarity index 88% rename from iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java rename to iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java index 7ab1f978f95b..eaeb66fc0e1c 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsWritePolicy.java @@ -19,17 +19,17 @@ package org.apache.iceberg.mr.hive.stats; -import org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.ColStatsWrite; +import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.Facts; import org.junit.Test; -import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.MERGE; -import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.REPLACE; -import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.SKIP; -import static org.apache.iceberg.mr.hive.stats.ColStatsWritePolicy.resolve; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.MERGE; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.REPLACE; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.SKIP; +import static org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter.WritePolicy.resolve; import static org.junit.Assert.assertEquals; /** Every case of the decision, stated as one, without a session to make it in. */ -public class TestColStatsWritePolicy { +public class TestIcebergColStatsWritePolicy { @Test public void analyzeOfTheWholeTableReplacesEveryPartition() { @@ -60,8 +60,10 @@ public void aWriteThatProducedEveryRowReplaces() { } @Test - public void aWriteThatReachedSomePartitionsKeepsTheOthers() { - assertEquals(MERGE, resolve(partitionLevel().build())); + public void aWriteThatReplacedSomePartitionsStandsForThem() { + assertEquals(MERGE, resolve(partitionLevel().replacePartitions().build())); + // a plain insert only added to partitions, and rows of part of a partition describe none of it + assertEquals(SKIP, resolve(partitionLevel().build())); } @Test @@ -218,8 +220,8 @@ private Builder statsAccurate() { return this; } - private ColStatsWrite build() { - return new ColStatsWrite(tableWideStats, keepsStatsPerPartition, partitioned, () -> statsAccurate, + private Facts build() { + return new Facts(tableWideStats, keepsStatsPerPartition, partitioned, () -> statsAccurate, analyze, analyzePartition, compaction, majorCompaction, fullTableRewrite, singlePartitionRewrite, holdsOnlyAddedRows, emptySnapshot, wroteNoRows, replacePartitions); } diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 8b4963ceb7e7..bfd4698b73b3 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -57,14 +57,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -73,17 +73,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -93,14 +93,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -164,14 +164,14 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -180,17 +180,17 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int) outputColumnNames: strcol, intcol, pcol - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(length(strcol)), avg(COALESCE(length(strcol),0)), count(1), count(strcol), compute_bit_vector_hll(strcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary) Execution mode: vectorized Reducer 2 @@ -200,14 +200,14 @@ STAGE PLANS: aggregations: max(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), min(VALUE._col5), max(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10), count(VALUE._col11), compute_bit_vector_hll(VALUE._col12) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col0,0)) (type: bigint), COALESCE(_col1,0) (type: double), (_col2 - _col3) (type: bigint), COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary), 'LONG' (type: string), UDFToLong(_col5) (type: bigint), UDFToLong(_col6) (type: bigint), (_col2 - _col7) (type: bigint), COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary), 'LONG' (type: string), UDFToLong(_col9) (type: bigint), UDFToLong(_col10) (type: bigint), (_col2 - _col11) (type: bigint), COALESCE(ndv_compute_bit_vector(_col12),0) (type: bigint), _col12 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index c90b08189682..50bd74b91a9a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -512,14 +512,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v4 Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -554,14 +554,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -598,17 +598,17 @@ STAGE PLANS: alias: ice_orc.tag_v1 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v1 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 396 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -739,17 +739,17 @@ STAGE PLANS: alias: ice_orc.tag_v4 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -785,17 +785,17 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -926,7 +926,7 @@ POSTHOOK: type: SHOW COMPACTIONS CompactionId Database Table Partition Type State Worker host Worker Enqueue Time Start Time Duration(ms) HadoopJobId Error message Initiator host Initiator Pool name TxnId Next TxnId Commit Time Highest WriteId #Masked# default ice_orc company_id=100/dept_id=1 MAJOR succeeded #Masked# manual iceberg 0 0 0 --- #Masked# default ice_orc company_id=100/dept_id=2 MAJOR succeeded #Masked# manual iceberg 0 0 0 --- -#Masked# default ice_orc company_id=null/dept_id=null MAJOR refused #Masked# manual iceberg 0 0 0 --- +#Masked# default ice_orc company_id=__HIVE_DEFAULT_PARTITION__/dept_id=__HIVE_DEFAULT_PARTITION__ MAJOR refused #Masked# manual iceberg 0 0 0 --- #Masked# default ice_orc --- MAJOR succeeded #Masked# manual iceberg 0 0 0 --- PREHOOK: query: select `partition`, spec_id, content, record_count from default.ice_orc.files @@ -965,14 +965,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1386 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1008,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 2352 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index b0786216df44..fb8d8fcaa93e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -79,6 +79,7 @@ import org.apache.hadoop.mapred.OutputFormat; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -330,13 +331,16 @@ default Map> getAggrBasicStatsFor(org.apache.hadoop. } /** - * Set column stats for non-native tables + * Persists the column statistics a gather computed. They are pulled batch by batch, so the whole + * of a large table's statistics is never held at once. * @param table table object - * @param colStats list of ColumnStatistics objects - * @return true if operation is successful + * @param colStats the computed statistics, one entry for the table or one per partition + * @return whether the stored statistics now describe the table */ - default boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, List colStats) { - return false; + default boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table table, + Iterator colStats) { + throw new UnsupportedOperationException( + this.getClass().getName() + " does not support column statistics"); } /** @@ -630,7 +634,6 @@ default Function partitionNameResolver( throw new UnsupportedOperationException(getClass().getName() + " does not name partitions"); } - /** * Creates a DynamicPartitionCtx instance that will be set up by the storage handler itself. Useful for non-native * tables where partitions are not handled by Hive, and sorting is required in a custom way before writing the table. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java index 1f8fe6736b42..438d21b9ca14 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java @@ -176,8 +176,8 @@ public void run() { try { Table table = partish.getTable(); if (partish.getPartition() != null) { - // the storage handler maintains the statistics of all partitions as a whole; a partition-scoped - // ANALYZE is rejected at compile time (ErrorMsg.ANALYZE_PARTITION_NON_NATIVE) + // the storage handler maintains basic statistics for the table as a whole, so no + // statement produces a partition partish for it here throw new IllegalStateException("Partition-scoped statistics collection is not supported for " + table.getFullyQualifiedName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 7c357b680409..9622d79ba40e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -139,8 +139,8 @@ public BasicStatsProcessor(Partish partish, BasicStatsWork work, boolean followe this.followedColStats1 = followedColStats2; Table table = partish.getTable(); - // table scope only: a partition-scoped ANALYZE is rejected at compile time for tables with - // non-native partitioning (ErrorMsg.ANALYZE_PARTITION_NON_NATIVE) + // a non-native table without native partition support reaches this task with partitions, + // which the handler's table-level statistics do not describe if (table.isNonNative() && table.getStorageHandler().canProvideBasicStatistics() && partish.getPartition() == null) { this.providedBasicStats = table.getStorageHandler().computeBasicStatistics(table); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 9adb12bbc5c5..1f67f511c1bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -21,9 +21,13 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; import java.util.function.Function; import java.util.stream.Collectors; @@ -221,9 +225,20 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce // the metastore holds only the table's statistics; a branch has nowhere to store its own return 0; } - // a storage handler persists all the statistics in a single batch - long maxNumStats = useStorageHandler ? - Long.MAX_VALUE : conf.getLongVar(HiveConf.ConfVars.HIVE_STATS_MAX_NUM_STATS); + long maxNumStats = conf.getLongVar(HiveConf.ConfVars.HIVE_STATS_MAX_NUM_STATS); + if (useStorageHandler) { + // the handler pulls the statistics batch by batch and persists them in one pass, so the + // whole of a large table's statistics is never held here at once + Iterator stats = columnStatsIterator(tbl, maxNumStats); + if (stats.hasNext()) { + boolean success = tbl.getStorageHandler().setColStatistics(tbl, stats); + // COLUMN_STATS_ACCURATE describes the table, so a branch write leaves it alone + if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary()) && tbl.getSnapshotRef() == null) { + setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); + } + } + return 0; + } while (!done) { List colStats = new ArrayList<>(); @@ -247,21 +262,44 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce } start = System. currentTimeMillis(); - if (useStorageHandler) { - boolean success = tbl.getStorageHandler().setColStatistics(tbl, colStats); - // COLUMN_STATS_ACCURATE describes the table, so a branch write leaves it alone - if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary()) && tbl.getSnapshotRef() == null) { - setOrRemoveColumnStatsAccurateProperty(db, tbl, colStatDesc.getColName(), success); - } - } else { - db.setPartitionColumnStatistics(request); - } + db.setPartitionColumnStatistics(request); end = System.currentTimeMillis(); LOG.info("Time taken to update " + colStats.size() + " stats : " + ((end - start)/1000F) + " seconds."); } return 0; } + /** The computed statistics, fetched and decoded a batch at a time as they are pulled. */ + private Iterator columnStatsIterator(Table tbl, long maxNumStats) { + return new Iterator<>() { + private final Queue batch = new ArrayDeque<>(); + private boolean done = false; + + @Override + public boolean hasNext() { + while (batch.isEmpty() && !done) { + List next = new ArrayList<>(); + try { + done = constructColumnStatsFromPackedRows(tbl, next, maxNumStats); + } catch (HiveException | IOException | MetaException e) { + // a pull the interface cannot declare checked exceptions for; + throw new RuntimeException("Failed to fetch computed column statistics", e); + } + batch.addAll(next); + } + return !batch.isEmpty(); + } + + @Override + public ColumnStatistics next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return batch.poll(); + } + }; + } + @Override public void setDpPartSpecs(Collection dpPartSpecs) { } From f3c31ca27aef6dd2ea60f8549877ee0bb30dcc65 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 25 Aug 2026 21:50:06 +0300 Subject: [PATCH 09/15] HIVE-29829: A compaction of an older spec's partition stores no statistics Compacting a partition of an older spec rewrites its rows into partitions of the current one, and what it measured - the migrated rows alone - replaced the stored entry of the partition they landed in: max over an evolved table answered from the hundred migrated rows instead of the eleven hundred present. Only a compaction of a current-spec partition gathers now; the snapshot an old-spec compaction commits already invalidates the stored entries of every partition it touched. --- .../compaction/IcebergQueryCompactor.java | 23 ++++++---- .../compactor/TestIcebergCompactorOnTez.java | 43 +++++++++++++++++++ .../ql/parse/ColumnStatsSemanticAnalyzer.java | 14 ++---- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java index 5e3f4960eec1..7b80d658293b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java @@ -115,19 +115,27 @@ private String buildCompactionQuery(CompactorContext context, String compactTabl } } - // only a compaction that rewrote the whole table, or a whole partition of a table keeping - // statistics per partition, may store what it measured - boolean computeColStats = ci.isMajorCompaction() && (ci.partName == null ? + PartitionSpec spec; + try { + spec = (ci.partName == null) ? null : IcebergTableUtil.getPartitionSpec(icebergTable, ci.partName); + } catch (MetaException e) { + throw new HiveException(e); + } + + // gather only when the compaction reads something whole: the table, or one current-spec + // partition of a table keeping partition-level statistics. An older spec's partition is + // rewritten into partitions of the current spec and describes none of them whole. + boolean computeColStats = ci.isMajorCompaction() && (spec == null ? !icebergTable.spec().isPartitioned() : - IcebergTableUtil.isPartitionStats(icebergTable, conf)); + IcebergTableUtil.isPartitionStats(icebergTable, conf) && spec.specId() == icebergTable.spec().specId()); boolean genericStats = HiveConf.getBoolVar(conf, ConfVars.HIVE_COMPACTOR_GATHER_STATS); HiveConf.setBoolVar(conf, ConfVars.HIVE_STATS_COL_AUTOGATHER, genericStats && computeColStats); - String compactionQuery = (ci.partName == null) ? + String compactionQuery = (spec == null) ? buildFullTableCompactionQuery(compactTableName, conf, icebergTable, columnsList, fileSizePredicate, orderBy) : - buildPartitionCompactionQuery(ci, compactTableName, conf, icebergTable, + buildPartitionCompactionQuery(ci, compactTableName, conf, icebergTable, spec, columnsList, fileSizePredicate, orderBy); LOG.info("Compaction query: {}", compactionQuery); @@ -187,6 +195,7 @@ private String buildPartitionCompactionQuery( String compactTableName, HiveConf conf, Table icebergTable, + PartitionSpec spec, String columnsList, String fileSizePredicate, String orderBy) throws HiveException { @@ -195,10 +204,8 @@ private String buildPartitionCompactionQuery( HiveConf.setVar(conf, ConfVars.REWRITE_POLICY, RewritePolicy.PARTITION.name()); conf.set(IcebergCompactionService.PARTITION_NAME, ci.partName); - PartitionSpec spec; String partitionPredicate; try { - spec = IcebergTableUtil.getPartitionSpec(icebergTable, ci.partName); partitionPredicate = buildPartitionPredicate(ci, spec); } catch (MetaException e) { throw new HiveException(e); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java index c73a90c26805..048c4c05c596 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestIcebergCompactorOnTez.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -168,6 +169,48 @@ private int getFilesCount() throws Exception { return Integer.parseInt(res.getFirst()); } + @Test + public void testOldSpecPartitionCompactionKeepsCurrentSpecStatistics() throws Exception { + // Compacting a partition of an older spec rewrites its rows into partitions of the current + // one, so what the compaction measured describes no current partition whole. The statistics + // stored for the current partitions must survive it untouched, or an aggregate answered from + // them would describe only the migrated rows. + conf.setBoolVar(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_METADATA_QUERIES, true); + driver = DriverFactory.newDriver(conf); + try { + executeStatementOnDriver(String.format("create table %s " + + "(id int, c bigint, a int, b string) " + + "partitioned by spec(a) stored by iceberg stored as orc " + + "tblproperties ('compactor.threshold.min.input.files'='1', " + + "'compactor.threshold.target.size'='1500')", QUALIFIED_TABLE_NAME), driver); + + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (1, 0, 1, 'x7')", QUALIFIED_TABLE_NAME), driver); + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (2, 10, 1, 'x7')", QUALIFIED_TABLE_NAME), driver); + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (5, 3, 1, 'x7')", QUALIFIED_TABLE_NAME), driver); + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (6, 7, 1, 'x7')", QUALIFIED_TABLE_NAME), driver); + + executeStatementOnDriver(String.format("alter table %s set partition spec(b)", QUALIFIED_TABLE_NAME), driver); + + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (3, 50, 2, 'x7')", QUALIFIED_TABLE_NAME), driver); + executeStatementOnDriver(String.format("INSERT INTO %s VALUES (4, 100, 2, 'x7')", QUALIFIED_TABLE_NAME), driver); + executeStatementOnDriver(String.format("analyze table %s compute statistics for columns", + QUALIFIED_TABLE_NAME), driver); + + CompactorTestUtil.runCompaction(conf, DB_NAME, TABLE_NAME, CompactionType.MAJOR, false, "a=1"); + Assert.assertTrue(isCompactExist(new HiveMetaStoreClient(conf).showCompactions(), "a=1", + CompactionType.MAJOR, CompactionState.SUCCEEDED)); + + driver.run(String.format("select max(c) from %s where b = 'x7'", QUALIFIED_TABLE_NAME)); + List res = new ArrayList<>(); + driver.getFetchTask().fetch(res); + Assert.assertEquals(List.of("100"), res); + } finally { + conf.setBoolVar(HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_METADATA_QUERIES, false); + } + } + private List getAllRecords() throws Exception { driver.run(String.format("select * from %s order by id", QUALIFIED_TABLE_NAME)); List res = new ArrayList<>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 20b48f40208f..d96987f82080 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -199,17 +199,9 @@ private static CharSequence genPartitionPredicate(Table tbl, Map } private static CharSequence genPartitionClause(Table tbl, Map partSpec, HiveConf conf) { - boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); - - StringBuilder whereClause = new StringBuilder(genPartitionPredicate(tbl, partSpec, conf)); - - StringBuilder groupByClause = new StringBuilder(" group by ").append( - tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) - .collect(Collectors.joining(", ")) - ); - - // attach the predicate and group by to the return clause - return predPresent ? whereClause.append(groupByClause) : groupByClause; + return genPartitionPredicate(tbl, partSpec, conf) + " group by " + + tbl.getPartColNames().stream().map(col -> unparseIdentifier(col, conf)) + .collect(Collectors.joining(", ")); } /** From e387fcdea46383bc186cd72db5f999d397fad8ba Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 24 Aug 2026 15:54:00 +0300 Subject: [PATCH 10/15] HIVE-29834: Iceberg: Answer min/max/count aggregates from column statistics StatsOptimizer fetched partition column statistics straight from the metastore, which cannot hold them for an Iceberg table: PART_COL_STATS rows require a PARTITIONS row that Iceberg never creates. Aggregates over a partitioned Iceberg table therefore always executed, even with statistics available in Puffin files. Fetch through Hive.getAggrColStatsFor, which consults the storage handler, mirroring what getRowCnt already does for row counts. The statistics arrive aggregated over the pruned partitions rather than one entry per partition; the callers folded them with min, max or a sum, so merging first gives the same answer and the folds collapse. Accuracy is asked of the component that owns it. The metastore's marker only records Hive's own writes, while an Iceberg table may be written by other engines, so areColumnStatsUptoDate is added to HiveStorageHandler and answered from table metadata by the Iceberg handler. The default keeps the metastore marker, so other handlers are unchanged, and the Iceberg override defers to the metastore when hive.iceberg.stats.source says the statistics live there. A table that keeps statistics per partition has no table-wide verdict to give: they describe no partition in particular. The partitioned non-native path therefore asks only whether the table provides column statistics at all, and the aggregate - the one place told the partitions the scan pruned to - reports how many of them it found, which is what stops the rewrite from describing a subset. count(col) needs a row count as well as the column's null count, and read it from partition parameters, which a storage handler does not keep, so it gave up on every partitioned non-native table before reaching the column statistics at all. It now asks getRowCnt, which consults the handler and is given the pruned partitions; a count no handler can determine exactly, such as one covering delete files, still stops the rewrite. --- .../mr/hive/HiveIcebergStorageHandler.java | 10 + .../queries/positive/iceberg_part_colstats.q | 102 ++++ .../src/test/results/positive/col_stats.q.out | 18 +- .../positive/iceberg_part_colstats.q.out | 539 ++++++++++++++++++ .../hive/ql/metadata/HiveStorageHandler.java | 13 + .../hive/ql/optimizer/StatsOptimizer.java | 204 +++---- .../hadoop/hive/ql/stats/StatsUtils.java | 7 +- 7 files changed, 778 insertions(+), 115 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 5032037f7783..323cf0955097 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -723,6 +723,16 @@ public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsT return HiveMetaHook.ICEBERG.equals(getStatsSource()); } + @Override + public boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, String colName) { + if (canSetColStatistics(hmsTable)) { + return IcebergTableUtil.colStatsAccurate(hmsTable, conf); + } + // the metastore holds them, and its single row describes the table: a branch has none + return hmsTable.getSnapshotRef() == null && + StatsSetupConst.areColumnStatsUptoDate(hmsTable.getParameters(), colName); + } + @Override public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Iterator colStats) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q new file mode 100644 index 000000000000..0b4fc14da9f7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -0,0 +1,102 @@ +--! qt:replace:/(\s+Statistics\: Num rows\: \d+ Data size\:\s+)\S+(\s+Basic stats\: \S+ Column stats\: \S+)/$1#Masked#$2/ + +-- Column statistics kept per partition answer a query over the partitions it pruned to, and stop +-- answering only for the partitions a later write reached. + +set hive.explain.user=false; +set hive.compute.query.using.stats=true; +set hive.fetch.task.conversion=none; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c'); +analyze table ice_part_stats compute statistics for columns; + +-- answered from the statistics of the pruned partition alone +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- a write that reaches p=a only +insert into ice_part_stats values (11, 'a'); + +-- p=a describes itself no longer, so the query has to read it +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- the partitions that write never touched still answer from their statistics +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- an ANALYZE naming the written partition measures it again, and it answers from statistics once +-- more while the partitions carried across that ANALYZE keep the numbers they were computed with +analyze table ice_part_stats partition (p = 'a') compute statistics for columns; + +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- count(col) needs a row count as well as the column's null count, and a handler keeps no +-- partition parameters to read one from: it is asked of the table for the pruned partitions +explain +select count(id) from ice_part_stats where p = 'b'; + +select count(id) from ice_part_stats where p = 'b'; + +-- a query spanning a written and an untouched partition cannot be answered from a subset +explain +select max(id) from ice_part_stats where p in ('a', 'b'); + +select max(id) from ice_part_stats where p in ('a', 'b'); + +drop table ice_part_stats; + +-- statistics kept for the table as a whole describe no partition in particular, so a query over +-- one of them cannot be answered from them however fresh they are +set hive.iceberg.stats.collect.partlevel=false; + +create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b'); +analyze table ice_tbl_level compute statistics for columns; + +explain +select max(id) from ice_tbl_level where p = 'a'; + +select max(id) from ice_tbl_level where p = 'a'; + +drop table ice_tbl_level; + +set hive.iceberg.stats.collect.partlevel=true; + +-- with the statistics kept by the metastore there are no per-partition numbers to answer from +set hive.iceberg.stats.source=metastore; + +create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b'); + +explain +select max(id) from ice_part_stats_hms where p = 'a'; + +select max(id) from ice_part_stats_hms where p = 'a'; + +drop table ice_part_stats_hms; diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out index ef5c4410fbb6..50f7b4b6059b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out @@ -321,25 +321,9 @@ POSTHOOK: Input: default@tbl_ice_puffin POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. -Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Stage-0 Fetch Operator - limit:-1 - Stage-1 - Reducer 2 vectorized - File Output Operator [FS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_9] - Group By Operator [GBY_8] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(a)","max(c)"] - Select Operator [SEL_7] (rows=5 width=8) - Output:["a","c"] - TableScan [TS_0] (rows=5 width=8) - default@tbl_ice_puffin,tbl_ice_puffin,Tbl:COMPLETE,Col:COMPLETE,Output:["a","c"] + limit:1 PREHOOK: query: desc formatted tbl_ice_puffin C PREHOOK: type: DESCTABLE diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out new file mode 100644 index 000000000000..830acf5cabb6 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -0,0 +1,539 @@ +PREHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: analyze table ice_part_stats compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: default@ice_part_stats@p=b +PREHOOK: Output: default@ice_part_stats@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: default@ice_part_stats@p=b +POSTHOOK: Output: default@ice_part_stats@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_part_stats values (11, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (11, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: drop table ice_part_stats +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: drop table ice_part_stats +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: analyze table ice_tbl_level compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level@p=a +PREHOOK: Output: default@ice_tbl_level@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_tbl_level compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level@p=a +POSTHOOK: Output: default@ice_tbl_level@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_tbl_level + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_tbl_level +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: drop table ice_tbl_level +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats_hms + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_part_stats_hms +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: drop table ice_part_stats_hms +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index fb8d8fcaa93e..6501d36efb8e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.common.type.SnapshotContext; @@ -372,6 +373,18 @@ default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table tab return false; } + /** + * Whether the column statistics the handler holds still describe the table, so that a query may + * be answered from them rather than by reading the data. The metastore's accuracy marker only + * records what Hive itself wrote, while a handler's table may be written by other engines. + * @param table table object + * @param colName the column being asked about + * @return true if the statistics still describe the table + */ + default boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table table, String colName) { + return StatsSetupConst.areColumnStatsUptoDate(table.getParameters(), colName); + } + /** * Returns the row count of the table, letting queries like count(1) be answered from statistics. * @param hmsTable table object diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index bff675b0d98f..d0b00b39ab8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -22,12 +22,12 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -84,7 +84,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -501,33 +500,42 @@ else if (udaf instanceof GenericUDAFCount) { } else { Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) .getPartitions(); - for (Partition part : parts) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { - Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); + if (tbl.isNonNative()) { + // a handler holds no partition parameters, so its row counts are asked of the + // table, which is told the partitions this scan was pruned to + Long handlerRowCnt = getRowCnt(tsOp, tbl); + if (handlerRowCnt == null) { + Logger.debug("No exact row count for table : " + tbl.getTableName()); return null; } - long partRowCnt = Long.parseLong(part.getParameters().get( - StatsSetupConst.ROW_COUNT)); - rowCnt += partRowCnt; + rowCnt = handlerRowCnt; + } else { + for (Partition part : parts) { + if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { + Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); + return null; + } + long partRowCnt = Long.parseLong(part.getParameters().get( + StatsSetupConst.ROW_COUNT)); + rowCnt += partRowCnt; + } } - Collection> result = verifyAndGetPartColumnStats(hive, + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) - return null; - Long nullCnt = getNullcountFor(type, statData); - if (nullCnt == null) { - Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " - + "metadata optimizer for column : " + colName); - return null; - } else { - rowCnt -= nullCnt; - } + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + Long nullCnt = getNullcountFor(type, statData); + if (nullCnt == null) { + Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } + rowCnt -= nullCnt; } } oneRow.add(rowCnt); @@ -598,20 +606,18 @@ else if (udaf instanceof GenericUDAFCount) { LongSubType subType = LongSubType.valueOf(name); Long maxVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetHighValue()) { - continue; - } - long curVal = lstats.getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + LongColumnStatsData lstats = statData.getLongStats(); + if (lstats.isSetHighValue()) { + maxVal = lstats.getHighValue(); } if (maxVal != null) { oneRow.add(subType.cast(maxVal)); @@ -624,20 +630,18 @@ else if (udaf instanceof GenericUDAFCount) { DoubleSubType subType = DoubleSubType.valueOf(name); Double maxVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetHighValue()) { - continue; - } - double curVal = statData.getDoubleStats().getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + DoubleColumnStatsData dstats = statData.getDoubleStats(); + if (dstats.isSetHighValue()) { + maxVal = statData.getDoubleStats().getHighValue(); } if (maxVal != null) { oneRow.add(subType.cast(maxVal)); @@ -648,20 +652,18 @@ else if (udaf instanceof GenericUDAFCount) { } case Date: { Long maxVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetHighValue()) { - continue; - } - long curVal = dstats.getHighValue().getDaysSinceEpoch(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + DateColumnStatsData dstats = statData.getDateStats(); + if (dstats.isSetHighValue()) { + maxVal = dstats.getHighValue().getDaysSinceEpoch(); } if (maxVal != null) { oneRow.add(DateSubType.DAYS.cast(maxVal)); @@ -735,20 +737,18 @@ else if (udaf instanceof GenericUDAFCount) { LongSubType subType = LongSubType.valueOf(name); Long minVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetLowValue()) { - continue; - } - long curVal = lstats.getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + LongColumnStatsData lstats = statData.getLongStats(); + if (lstats.isSetLowValue()) { + minVal = lstats.getLowValue(); } if (minVal != null) { oneRow.add(subType.cast(minVal)); @@ -761,20 +761,18 @@ else if (udaf instanceof GenericUDAFCount) { DoubleSubType subType = DoubleSubType.valueOf(name); Double minVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetLowValue()) { - continue; - } - double curVal = statData.getDoubleStats().getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + DoubleColumnStatsData dstats = statData.getDoubleStats(); + if (dstats.isSetLowValue()) { + minVal = statData.getDoubleStats().getLowValue(); } if (minVal != null) { oneRow.add(subType.cast(minVal)); @@ -785,20 +783,18 @@ else if (udaf instanceof GenericUDAFCount) { } case Date: { Long minVal = null; - Collection> result = + List result = verifyAndGetPartColumnStats(hive, tbl, colName, parts); if (result == null) { return null; // logging inside } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetLowValue()) { - continue; - } - long curVal = dstats.getLowValue().getDaysSinceEpoch(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); + ColumnStatisticsData statData = validateSingleColStat(result); + if (statData == null) { + return null; + } + DateColumnStatsData dstats = statData.getDateStats(); + if (dstats.isSetLowValue()) { + minVal = dstats.getLowValue().getDaysSinceEpoch(); } if (minVal != null) { oneRow.add(DateSubType.DAYS.cast(minVal)); @@ -910,28 +906,42 @@ private ColumnStatisticsData validateSingleColStat(List sta return statObj.get(0).getStatsData(); } - private Collection> verifyAndGetPartColumnStats( + private List verifyAndGetPartColumnStats( Hive hive, Table tbl, String colName, Set parts) throws TException, LockException { - List partNames = new ArrayList(parts.size()); - for (Partition part : parts) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { - Logger.debug("Stats for part : " + part.getSpec() + " column " + colName - + " are not up to date."); + List partNames = new ArrayList<>(parts.size()); + // a storage handler holds no partition parameters, and one kept per partition describes no + // partition in particular: whether each still describes itself is answered by the aggregate + // below, which is told the partitions this query pruned to + if (tbl.isNonNative()) { + if (!StatsUtils.checkCanProvideColumnStats(tbl)) { + Logger.debug("Table : " + tbl.getTableName() + " provides no column statistics."); return null; } - partNames.add(part.getName()); + parts.forEach(part -> partNames.add(part.getName())); + } else { + for (Partition part : parts) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { + Logger.debug("Stats for part : " + part.getSpec() + " column " + colName + + " are not up to date."); + return null; + } + partNames.add(part.getName()); + } + } + // Aggregated rather than per partition: the callers fold these with min, max or a sum, so + // merging first gives the same answer. It also reaches a handler's own statistics, which + // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. + AggrStats aggrStats = hive.getAggrColStatsFor(tbl, Lists.newArrayList(colName), partNames, true); + if (aggrStats == null || aggrStats.getColStats() == null) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); + return null; } - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(hive.getConf(), tbl); - - Map> result = hive.getMSC().getPartitionColumnStatistics( - tbl.getDbName(), tbl.getTableName(), partNames, Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (result.size() != parts.size()) { - Logger.debug("Received " + result.size() + " stats for " + parts.size() + " partitions"); + if (aggrStats.getPartsFound() != parts.size()) { + // a partition whose statistics are missing would leave the answer describing a subset + Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); return null; } - return result.values(); + return aggrStats.getColStats(); } private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 22df96e85ae4..e0a2b5f3d9cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -2067,7 +2067,12 @@ public static boolean areBasicStatsUptoDateForQueryAnswering(Table table, Map params, String colName) { - return checkCanProvideStats(table) && StatsSetupConst.areColumnStatsUptoDate(params, colName); + // a handler keeps its own statistics and knows what happened to them, including writes by + // other engines that never touched the metastore marker + return checkCanProvideStats(table) && ( + table.isNonNative() ? table.getStorageHandler().areColumnStatsUptoDate(table, colName) : + StatsSetupConst.areColumnStatsUptoDate(params, colName) + ); } /** From c240c0703239dad83015a0147e97e969ae05905e Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sun, 23 Aug 2026 12:45:14 +0300 Subject: [PATCH 11/15] HIVE-29834: Answer aggregates of an unpartitioned table from the handler's statistics An unpartitioned table's aggregates were fetched with the metastore client directly, which for a table whose statistics a handler keeps returns nothing: ColStatsProcessor writes the values to the handler and only the accuracy marker to the metastore. The accuracy check above the fetch therefore guarded a fetch that could not have served them, and min, max and count over an unpartitioned Iceberg table always read the data. Fetch through Hive.getTableColumnStatistics, which asks the handler when it can provide, mirroring what the partitioned path already does. A native table takes the same call it took before, resolving its snapshot and write id list the same way. The accuracy check now guards statistics that exist, which is what stops a write another engine made from being answered from numbers that predate it. The third of the three fetches took the first element without checking that there was one; a handler may legitimately return none. --- .../queries/positive/iceberg_part_colstats.q | 39 +++ .../positive/iceberg_part_colstats.q.out | 230 ++++++++++++++++++ .../hive/ql/optimizer/StatsOptimizer.java | 26 +- 3 files changed, 278 insertions(+), 17 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q index 0b4fc14da9f7..e01bc9b000cf 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -24,6 +24,11 @@ select max(id) from ice_part_stats where p = 'a'; -- a write that reaches p=a only insert into ice_part_stats values (11, 'a'); +-- a scan spanning the partition the write reached and one it did not has statistics for only part +-- of what it reads, which is what PARTIAL says +explain +select id from ice_part_stats where p in ('a', 'b'); + -- p=a describes itself no longer, so the query has to read it explain select max(id) from ice_part_stats where p = 'a'; @@ -65,6 +70,40 @@ select max(id) from ice_part_stats where p in ('a', 'b'); drop table ice_part_stats; +-- an unpartitioned table keeps its statistics in the same file, which the metastore never holds: +-- reaching them takes the handler, and only the accuracy check stands between a query and stale ones +create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_unpart values (1), (5), (9); +analyze table ice_unpart compute statistics for columns; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- an incremental gather keeps them describing the table, so it still answers +insert into ice_unpart values (11); + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- a write that records nothing, as another engine's would, leaves them behind: only the accuracy +-- check stands between the query and a value the table no longer holds +set hive.stats.autogather=false; +insert into ice_unpart values (20); +set hive.stats.autogather=true; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +drop table ice_unpart; + -- statistics kept for the table as a whole describe no partition in particular, so a query over -- one of them cannot be answered from them however fresh they are set hive.iceberg.stats.collect.partlevel=false; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out index 830acf5cabb6..e6408906ec8e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -71,6 +71,50 @@ POSTHOOK: query: insert into ice_part_stats values (11, 'a') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p) IN ('a', 'b') (type: boolean) + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + PREHOOK: query: explain select max(id) from ice_part_stats where p = 'a' PREHOOK: type: QUERY @@ -313,6 +357,192 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@ice_part_stats POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: insert into ice_unpart values (1), (5), (9) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (1), (5), (9) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: analyze table ice_unpart compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_unpart compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_unpart values (11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: insert into ice_unpart values (20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +20 +PREHOOK: query: drop table ice_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: drop table ice_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart PREHOOK: query: create external table ice_tbl_level (id bigint, p string) partitioned by spec (p) stored by iceberg tblproperties ('format-version'='2') diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index d0b00b39ab8d..bb7bc3bddf03 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -21,7 +21,6 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -389,9 +388,6 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { List oneRow = new ArrayList(); - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(pctx.getConf(), tbl); - for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { if (aggr.getDistinct()) { // our stats for NDV is approx, not accurate. @@ -481,10 +477,7 @@ else if (udaf instanceof GenericUDAFCount) { } List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); + hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); if (stats.isEmpty()) { Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); return null; @@ -551,10 +544,7 @@ else if (udaf instanceof GenericUDAFCount) { } List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); + hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); if (stats.isEmpty()) { Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); return null; @@ -688,11 +678,13 @@ else if (udaf instanceof GenericUDAFCount) { + " are not up to date."); return null; } - ColumnStatisticsData statData = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null) - .get(0).getStatsData(); + List minStats = + hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); + if (minStats.isEmpty()) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); + return null; + } + ColumnStatisticsData statData = minStats.get(0).getStatsData(); String name = colDesc.getTypeString().toUpperCase(); switch (type) { case Integer: { From 102fe221ae3c5b8cca4564ca5e950f989123bfc3 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sun, 23 Aug 2026 12:45:14 +0300 Subject: [PATCH 12/15] HIVE-29834: Fetch the column statistics of every aggregate at once The statistics of one partition carry every column of it, and the fetch was made once per aggregate, so a query over a thousand columns opened the statistics file a thousand times and discarded all but one column of each partition it had just deserialised. Collect the columns the aggregates read before walking them and fetch once for all of them, leaving each aggregate a lookup. A native table keeps its check of each column against each partition's own marker; only the fetch is shared. --- .../hive/ql/optimizer/StatsOptimizer.java | 133 +++++++++--------- 1 file changed, 63 insertions(+), 70 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index bb7bc3bddf03..4e42408cf7c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -85,6 +85,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.List; import java.util.Map; import java.util.Set; @@ -388,6 +390,14 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { List oneRow = new ArrayList(); + // Every aggregate of one query asks the same partitions about a column of the same table, + // and the statistics of one partition carry every column, so asking once for all of them + // reads what a thousand aggregates would have read a thousand times. + Set prunedParts = tbl.isPartitioned() ? + pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions() : null; + Map partColStats = prunedParts == null ? null : + verifyAndGetPartColumnStats(hive, tbl, aggregateColumns(pgbyOp, exprMap), prunedParts); + for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { if (aggr.getDistinct()) { // our stats for NDV is approx, not accurate. @@ -491,8 +501,7 @@ else if (udaf instanceof GenericUDAFCount) { rowCnt -= nullCnt; } } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) - .getPartitions(); + Set parts = prunedParts; if (tbl.isNonNative()) { // a handler holds no partition parameters, so its row counts are asked of the // table, which is told the partitions this scan was pruned to @@ -513,14 +522,9 @@ else if (udaf instanceof GenericUDAFCount) { rowCnt += partRowCnt; } } - List result = verifyAndGetPartColumnStats(hive, - tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } Long nullCnt = getNullcountFor(type, statData); if (nullCnt == null) { @@ -588,22 +592,16 @@ else if (udaf instanceof GenericUDAFCount) { return null; } } else { - Set parts = pctx.getPrunedPartitions( - tsOp.getConf().getAlias(), tsOp).getPartitions(); + Set parts = prunedParts; String name = colDesc.getTypeString().toUpperCase(); switch (type) { case Integer: { LongSubType subType = LongSubType.valueOf(name); Long maxVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } LongColumnStatsData lstats = statData.getLongStats(); if (lstats.isSetHighValue()) { @@ -620,14 +618,9 @@ else if (udaf instanceof GenericUDAFCount) { DoubleSubType subType = DoubleSubType.valueOf(name); Double maxVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } DoubleColumnStatsData dstats = statData.getDoubleStats(); if (dstats.isSetHighValue()) { @@ -642,14 +635,9 @@ else if (udaf instanceof GenericUDAFCount) { } case Date: { Long maxVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } DateColumnStatsData dstats = statData.getDateStats(); if (dstats.isSetHighValue()) { @@ -722,21 +710,16 @@ else if (udaf instanceof GenericUDAFCount) { return null; } } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions(); + Set parts = prunedParts; String name = colDesc.getTypeString().toUpperCase(); switch(type) { case Integer: { LongSubType subType = LongSubType.valueOf(name); Long minVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } LongColumnStatsData lstats = statData.getLongStats(); if (lstats.isSetLowValue()) { @@ -753,14 +736,9 @@ else if (udaf instanceof GenericUDAFCount) { DoubleSubType subType = DoubleSubType.valueOf(name); Double minVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } DoubleColumnStatsData dstats = statData.getDoubleStats(); if (dstats.isSetLowValue()) { @@ -775,14 +753,9 @@ else if (udaf instanceof GenericUDAFCount) { } case Date: { Long minVal = null; - List result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - ColumnStatisticsData statData = validateSingleColStat(result); + ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); if (statData == null) { - return null; + return null; // logging inside } DateColumnStatsData dstats = statData.getDateStats(); if (dstats.isSetLowValue()) { @@ -887,19 +860,36 @@ else if (udaf instanceof GenericUDAFCount) { } } - private ColumnStatisticsData validateSingleColStat(List statObj) { - if (statObj.size() > 1) { - Logger.error("More than one stat for a single column!"); - return null; - } else if (statObj.isEmpty()) { - Logger.debug("No stats for some partition and column"); + + /** The columns the aggregates read, which are the ones statistics have to be fetched for. */ + private static List aggregateColumns(GroupByOperator pgbyOp, Map exprMap) { + return pgbyOp.getConf().getAggregators().stream() + .filter(aggr -> !aggr.getParameters().isEmpty()) + .map(aggr -> aggr.getParameters().get(0)) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> exprMap.get(((ExprNodeColumnDesc) desc).getColumn())) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> ((ExprNodeColumnDesc) desc).getColumn()) + .distinct() + .collect(Collectors.toList()); + } + + /** One column's statistics out of what was fetched for all of them. */ + private ColumnStatisticsData partColumnStat(Map partColStats, + Table tbl, String colName) { + ColumnStatisticsObj stat = partColStats == null ? null : partColStats.get(colName); + if (stat == null) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); return null; } - return statObj.get(0).getStatsData(); + return stat.getStatsData(); } - private List verifyAndGetPartColumnStats( - Hive hive, Table tbl, String colName, Set parts) throws TException, LockException { + private Map verifyAndGetPartColumnStats( + Hive hive, Table tbl, List colNames, Set parts) throws TException, LockException { + if (colNames.isEmpty()) { + return null; + } List partNames = new ArrayList<>(parts.size()); // a storage handler holds no partition parameters, and one kept per partition describes no // partition in particular: whether each still describes itself is answered by the aggregate @@ -912,10 +902,12 @@ private List verifyAndGetPartColumnStats( parts.forEach(part -> partNames.add(part.getName())); } else { for (Partition part : parts) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { - Logger.debug("Stats for part : " + part.getSpec() + " column " + colName - + " are not up to date."); - return null; + for (String colName : colNames) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { + Logger.debug("Stats for part : " + part.getSpec() + " column " + colName + + " are not up to date."); + return null; + } } partNames.add(part.getName()); } @@ -923,9 +915,9 @@ private List verifyAndGetPartColumnStats( // Aggregated rather than per partition: the callers fold these with min, max or a sum, so // merging first gives the same answer. It also reaches a handler's own statistics, which // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. - AggrStats aggrStats = hive.getAggrColStatsFor(tbl, Lists.newArrayList(colName), partNames, true); + AggrStats aggrStats = hive.getAggrColStatsFor(tbl, colNames, partNames, true); if (aggrStats == null || aggrStats.getColStats() == null) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); + Logger.debug("No stats for " + tbl.getTableName() + " columns " + colNames); return null; } if (aggrStats.getPartsFound() != parts.size()) { @@ -933,7 +925,8 @@ private List verifyAndGetPartColumnStats( Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); return null; } - return aggrStats.getColStats(); + return aggrStats.getColStats().stream().collect( + Collectors.toMap(ColumnStatisticsObj::getColName, Function.identity(), (first, second) -> second)); } private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { From e5033e464624a106531e286c5864c09d5b810131 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 24 Aug 2026 00:17:19 +0300 Subject: [PATCH 13/15] HIVE-29834: Answer aggregates from a handler's column statistics in one fetch The rewrite asks for every aggregate's column at once instead of once per aggregate, and only when an aggregate it can answer actually asks, so a query holding a sum or a count(distinct) pays nothing. A scan of an unpartitioned table is answered the same way, and one pruned to no partitions folds to the identity of each aggregate rather than declining. An unpruned scan of a table whose handler keeps whole-table statistics is answered from those directly: they describe exactly the rows read, so there is nothing to aggregate. The row count of an unpartitioned scan is asked of the table rather than read off its parameters, which describe the current snapshot however the scan resolved a branch, a tag or a version. --- .../queries/positive/iceberg_part_colstats.q | 11 + .../positive/iceberg_part_colstats.q.out | 72 ++- .../hive/ql/optimizer/StatsOptimizer.java | 522 ++++++++---------- .../test/queries/clientpositive/stats_part.q | 3 + .../clientpositive/llap/stats_part.q.out | 27 + 5 files changed, 334 insertions(+), 301 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q index e01bc9b000cf..147e373c18b2 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -115,6 +115,17 @@ stored by iceberg tblproperties ('format-version'='2'); insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b'); analyze table ice_tbl_level compute statistics for columns; +-- a scan reading every partition reads the whole table, which is what they do describe +explain +select max(id) from ice_tbl_level; + +select max(id) from ice_tbl_level; + +explain +select count(id) from ice_tbl_level; + +select count(id) from ice_tbl_level; + explain select max(id) from ice_tbl_level where p = 'a'; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out index e6408906ec8e..35dc90b880d7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -578,6 +578,64 @@ POSTHOOK: Output: default@ice_tbl_level@p=a POSTHOOK: Output: default@ice_tbl_level@p=b POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain +select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: explain +select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 +PREHOOK: query: explain select max(id) from ice_tbl_level where p = 'a' PREHOOK: type: QUERY PREHOOK: Input: default@ice_tbl_level @@ -604,21 +662,21 @@ STAGE PLANS: TableScan alias: ice_tbl_level filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: id (type: bigint) outputColumnNames: id - Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: max(id) - minReductionHashAggr: 0.5 + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: bigint) Execution mode: vectorized Reducer 2 @@ -628,10 +686,10 @@ STAGE PLANS: aggregations: max(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 4e42408cf7c2..a6a7de0a84ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -22,11 +22,21 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -48,7 +58,6 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.SemanticRule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -78,7 +87,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,7 +100,6 @@ import java.util.Set; import java.util.Stack; - /** There is a set of queries which can be answered entirely from statistics stored in metastore. * Examples of such queries are count(*), count(a), max(a), min(b) etc. Hive already collects * these basic statistics for query planning purposes. These same statistics can be used to @@ -215,7 +222,7 @@ private StatType getType(String origType) { return StatType.Unsupported; } - private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { + private Long getNullCountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : @@ -235,6 +242,22 @@ private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { } } + /** + * The statistics of no rows: nothing counted, no low or high value. The branches below already + * fold that to the right answer - zero for a count, NULL for a min or a max. + */ + private static ColumnStatisticsData emptyColStats(StatType type) { + return switch (type) { + case Integer -> ColumnStatisticsData.longStats(new LongColumnStatsData()); + case Double -> ColumnStatisticsData.doubleStats(new DoubleColumnStatsData()); + case String -> ColumnStatisticsData.stringStats(new StringColumnStatsData()); + case Boolean -> ColumnStatisticsData.booleanStats(new BooleanColumnStatsData()); + case Binary -> ColumnStatisticsData.binaryStats(new BinaryColumnStatsData()); + case Date -> ColumnStatisticsData.dateStats(new DateColumnStatsData()); + default -> null; + }; + } + private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); int numCols = gbyDesc.getOutputColumnNames().size(); @@ -393,10 +416,10 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { // Every aggregate of one query asks the same partitions about a column of the same table, // and the statistics of one partition carry every column, so asking once for all of them // reads what a thousand aggregates would have read a thousand times. - Set prunedParts = tbl.isPartitioned() ? - pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions() : null; - Map partColStats = prunedParts == null ? null : - verifyAndGetPartColumnStats(hive, tbl, aggregateColumns(pgbyOp, exprMap), prunedParts); + PrunedPartitionList prunedList = tbl.isPartitioned() ? + pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) : null; + ScanColStats scanColStats = + new ScanColStats(hive, tbl, aggregateColumns(pgbyOp, exprMap), prunedList); for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { if (aggr.getDistinct()) { @@ -475,24 +498,19 @@ else if (udaf instanceof GenericUDAFCount) { String colName = desc.getColumn(); StatType type = getType(desc.getTypeString()); if (!tbl.isPartitioned()) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(tbl, tbl.getParameters())) { - Logger.debug("Stats for table : " + tbl.getTableName() + " are not up to date."); + // asked of the table, not read off its parameters: those describe the current + // snapshot, while the column statistics below answer for the one this scan reads + Long tableRowCnt = getRowCnt(tsOp, tbl); + if (tableRowCnt == null) { + Logger.debug("No exact row count for table : " + tbl.getTableName()); return null; } - rowCnt = Long.valueOf(tbl.getProperty(StatsSetupConst.ROW_COUNT)); - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; - } - - List stats = - hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + rowCnt = tableRowCnt; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside } - Long nullCnt = getNullcountFor(type, stats.get(0).getStatsData()); + Long nullCnt = getNullCountFor(type, statData); if (null == nullCnt) { Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + "metadata optimizer for column : " + colName); @@ -501,7 +519,6 @@ else if (udaf instanceof GenericUDAFCount) { rowCnt -= nullCnt; } } else { - Set parts = prunedParts; if (tbl.isNonNative()) { // a handler holds no partition parameters, so its row counts are asked of the // table, which is told the partitions this scan was pruned to @@ -512,7 +529,7 @@ else if (udaf instanceof GenericUDAFCount) { } rowCnt = handlerRowCnt; } else { - for (Partition part : parts) { + for (Partition part : prunedList.getPartitions()) { if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); return null; @@ -522,11 +539,11 @@ else if (udaf instanceof GenericUDAFCount) { rowCnt += partRowCnt; } } - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); if (statData == null) { return null; // logging inside } - Long nullCnt = getNullcountFor(type, statData); + Long nullCnt = getNullCountFor(type, statData); if (nullCnt == null) { Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + "metadata optimizer for column : " + colName); @@ -540,240 +557,67 @@ else if (udaf instanceof GenericUDAFCount) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); - if(!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; - } - - List stats = - hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside + } + String name = colDesc.getTypeString().toUpperCase(); + switch (type) { + case Integer: { + LongColumnStatsData lstats = statData.getLongStats(); + oneRow.add(lstats.isSetHighValue() ? + LongSubType.valueOf(name).cast(lstats.getHighValue()) : null); + break; } - ColumnStatisticsData statData = stats.get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetHighValue()) { - oneRow.add(subType.cast(lstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetHighValue()) { - oneRow.add(subType.cast(dstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetHighValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getHighValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: - // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; + case Double: { + DoubleColumnStatsData dstats = statData.getDoubleStats(); + oneRow.add(dstats.isSetHighValue() ? + DoubleSubType.valueOf(name).cast(dstats.getHighValue()) : null); + break; } - } else { - Set parts = prunedParts; - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long maxVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetHighValue()) { - maxVal = lstats.getHighValue(); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(maxVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double maxVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetHighValue()) { - maxVal = statData.getDoubleStats().getHighValue(); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - case Date: { - Long maxVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetHighValue()) { - maxVal = dstats.getHighValue().getDaysSinceEpoch(); - } - if (maxVal != null) { - oneRow.add(DateSubType.DAYS.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - default: - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; + case Date: { + DateColumnStatsData dstats = statData.getDateStats(); + oneRow.add(dstats.isSetHighValue() ? + DateSubType.DAYS.cast(dstats.getHighValue().getDaysSinceEpoch()) : null); + break; } + default: + Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } } else if (udaf instanceof GenericUDAFMin) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); - if (!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside + } + String name = colDesc.getTypeString().toUpperCase(); + switch (type) { + case Integer: { + LongColumnStatsData lstats = statData.getLongStats(); + oneRow.add(lstats.isSetLowValue() ? + LongSubType.valueOf(name).cast(lstats.getLowValue()) : null); + break; } - List minStats = - hive.getTableColumnStatistics(tbl, Lists.newArrayList(colName), true); - if (minStats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + case Double: { + DoubleColumnStatsData dstats = statData.getDoubleStats(); + oneRow.add(dstats.isSetLowValue() ? + DoubleSubType.valueOf(name).cast(dstats.getLowValue()) : null); + break; } - ColumnStatisticsData statData = minStats.get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetLowValue()) { - oneRow.add(subType.cast(lstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetLowValue()) { - oneRow.add(subType.cast(dstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetLowValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getLowValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - } - } else { - Set parts = prunedParts; - String name = colDesc.getTypeString().toUpperCase(); - switch(type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long minVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetLowValue()) { - minVal = lstats.getLowValue(); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double minVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetLowValue()) { - minVal = statData.getDoubleStats().getLowValue(); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Date: { - Long minVal = null; - ColumnStatisticsData statData = partColumnStat(partColStats, tbl, colName); - if (statData == null) { - return null; // logging inside - } - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetLowValue()) { - minVal = dstats.getLowValue().getDaysSinceEpoch(); - } - if (minVal != null) { - oneRow.add(DateSubType.DAYS.cast(minVal)); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - + case Date: { + DateColumnStatsData dstats = statData.getDateStats(); + oneRow.add(dstats.isSetLowValue() ? + DateSubType.DAYS.cast(dstats.getLowValue().getDaysSinceEpoch()) : null); + break; } + default: + Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } } else { // Unsupported aggregation. Logger.debug("Unsupported aggregation for metadata optimizer: " @@ -860,7 +704,6 @@ else if (udaf instanceof GenericUDAFCount) { } } - /** The columns the aggregates read, which are the ones statistics have to be fetched for. */ private static List aggregateColumns(GroupByOperator pgbyOp, Map exprMap) { return pgbyOp.getConf().getAggregators().stream() @@ -874,59 +717,150 @@ private static List aggregateColumns(GroupByOperator pgbyOp, Map partColStats, - Table tbl, String colName) { - ColumnStatisticsObj stat = partColStats == null ? null : partColStats.get(colName); - if (stat == null) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; + /** + * The statistics of the columns a scan's aggregates read, fetched for every column when the + * first of them asks. An aggregate this rewrite cannot answer declines the query before + * asking, so a query holding one pays for nothing. Answers for a scan of a partitioned table. + */ + private static final class ScanColStats { + private final Hive hive; + private final Table tbl; + private final List colNames; + private final PrunedPartitionList prunedList; + private Map colStatsByName; + private boolean fetched; + + ScanColStats(Hive hive, Table tbl, List colNames, PrunedPartitionList prunedList) { + this.hive = hive; + this.tbl = tbl; + this.colNames = colNames; + this.prunedList = prunedList; } - return stat.getStatsData(); - } - private Map verifyAndGetPartColumnStats( - Hive hive, Table tbl, List colNames, Set parts) throws TException, LockException { - if (colNames.isEmpty()) { - return null; - } - List partNames = new ArrayList<>(parts.size()); - // a storage handler holds no partition parameters, and one kept per partition describes no - // partition in particular: whether each still describes itself is answered by the aggregate - // below, which is told the partitions this query pruned to - if (tbl.isNonNative()) { - if (!StatsUtils.checkCanProvideColumnStats(tbl)) { - Logger.debug("Table : " + tbl.getTableName() + " provides no column statistics."); + /** + * One column's statistics. A scan pruned to no partitions reads no rows, and the statistics + * of no rows are the empty ones: nothing counted, and no least or greatest to name. + */ + ColumnStatisticsData statsFor(String colName, StatType type) throws HiveException { + if (prunedList != null && prunedList.getPartitions().isEmpty()) { + return emptyColStats(type); + } + if (!fetched) { + fetched = true; + colStatsByName = prunedList == null ? tableColStats() : verifyAndFetch(); + } + ColumnStatisticsObj stat = colStatsByName == null ? null : colStatsByName.get(colName); + if (stat == null) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); return null; } - parts.forEach(part -> partNames.add(part.getName())); - } else { - for (Partition part : parts) { - for (String colName : colNames) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { - Logger.debug("Stats for part : " + part.getSpec() + " column " + colName - + " are not up to date."); - return null; + return stat.getStatsData(); + } + + /** + * Whether the table's own statistics answer for this scan: it keeps them for the table as + * a whole, and the scan reads every partition. They then describe exactly the rows read. + */ + private boolean answeredByTheTable() { + return !StatsUtils.isPartitionStats(tbl, hive.getConf()) && + prunedList.getReferredPartCols().isEmpty() && !prunedList.hasUnknownPartitions(); + } + + /** The table's own statistics, taken only while they still describe it. */ + private Map tableColStats() throws HiveException { + for (String colName : colNames) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { + Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName + + " are not up to date."); + return null; + } + } + return byColumnName(hive.getTableColumnStatistics(tbl, colNames, true)); + } + + /** What the scan's partitions hold for every column asked about, or null to decline. */ + private Map verifyAndFetch() throws HiveException { + Set parts = prunedList.getPartitions(); + List partNames = new ArrayList<>(parts.size()); + // a storage handler holds no partition parameters, and one kept per partition describes no + // partition in particular: whether each still describes itself is answered by the aggregate + // below, which is told the partitions this query pruned to + if (tbl.isNonNative()) { + if (!StatsUtils.checkCanProvideColumnStats(tbl)) { + Logger.debug("Table : " + tbl.getTableName() + " provides no column statistics."); + return null; + } + if (answeredByTheTable()) { + return tableColStats(); + } + parts.forEach(part -> partNames.add(part.getName())); + } else { + for (Partition part : parts) { + for (String colName : colNames) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering( + part.getTable(), part.getParameters(), colName)) { + Logger.debug("Stats for part : " + part.getSpec() + " column " + colName + + " are not up to date."); + return null; + } } + partNames.add(part.getName()); } - partNames.add(part.getName()); } + // Aggregated rather than per partition: the callers fold these with min, max or a sum, so + // merging first gives the same answer. A handler aggregates its own statistics, which + // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. + AggrStats aggrStats; + try { + aggrStats = tbl.isNonNative() + ? tbl.getStorageHandler().getAggrColStatsFor(tbl, colNames, partNames) + : exactAggrColStats(partNames); + } catch (MetaException e) { + throw new HiveException(e); + } + if (aggrStats == null || aggrStats.getColStats() == null) { + Logger.debug("No stats for " + tbl.getTableName() + " columns " + colNames); + return null; + } + if (aggrStats.getPartsFound() != parts.size()) { + // a partition whose statistics are missing would leave the answer describing a subset + Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); + return null; + } + return byColumnName(aggrStats.getColStats()); } - // Aggregated rather than per partition: the callers fold these with min, max or a sum, so - // merging first gives the same answer. It also reaches a handler's own statistics, which - // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. - AggrStats aggrStats = hive.getAggrColStatsFor(tbl, colNames, partNames, true); - if (aggrStats == null || aggrStats.getColStats() == null) { - Logger.debug("No stats for " + tbl.getTableName() + " columns " + colNames); - return null; + + /** + * Each partition fetched and folded the way a storage handler folds its own: the + * metastore's aggregate endpoint may serve a cached aggregate of a different partition + * set within its variance, which estimates a plan fine but must not answer a query. + */ + private AggrStats exactAggrColStats(List partNames) throws HiveException, MetaException { + Map> byPartition = hive.getPartitionColumnStatistics( + tbl.getDbName(), tbl.getTableName(), partNames, colNames, true); + List partStats = new ArrayList<>(); + byPartition.forEach((partitionName, statsObjs) -> { + // a partition counts as found only when it holds every column asked about + if (statsObjs.size() == colNames.size()) { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, tbl.getDbName(), tbl.getTableName()); + statsDesc.setPartName(partitionName); + partStats.add(new ColumnStatistics(statsDesc, statsObjs)); + } + }); + HiveConf conf = hive.getConf(); + List aggregated = MetaStoreServerUtils.aggrPartitionStats(partStats, + MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), tbl.getTableName(), + partNames, colNames, + partStats.size() == partNames.size(), + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION), + MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER)); + return new AggrStats(aggregated, partStats.size()); } - if (aggrStats.getPartsFound() != parts.size()) { - // a partition whose statistics are missing would leave the answer describing a subset - Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); - return null; + + private static Map byColumnName(List colStats) { + return colStats.stream().collect( + Collectors.toMap(ColumnStatisticsObj::getColName, Function.identity(), (first, second) -> second)); } - return aggrStats.getColStats().stream().collect( - Collectors.toMap(ColumnStatisticsObj::getColName, Function.identity(), (first, second) -> second)); } private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { diff --git a/ql/src/test/queries/clientpositive/stats_part.q b/ql/src/test/queries/clientpositive/stats_part.q index d0812e100781..1b94ed673867 100644 --- a/ql/src/test/queries/clientpositive/stats_part.q +++ b/ql/src/test/queries/clientpositive/stats_part.q @@ -45,6 +45,9 @@ explain select count(key) from stats_part; --select count(*) from stats_part where p = 100; explain select count(key) from stats_part where p > 100; --select count(*) from stats_part where p > 100; +-- pruned to no partitions, max has no greatest to name +explain select max(key) from stats_part; +select max(key) from stats_part; desc formatted stats_part; --explain insert into table stats_part partition(p=100) select distinct key, value from mysource where p == 100; diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index c474362fa244..a69590cc2f21 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -134,6 +134,33 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: explain select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: explain select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +NULL PREHOOK: query: desc formatted stats_part PREHOOK: type: DESCTABLE PREHOOK: Input: default@stats_part From c4ab0bcf9cbcdfafd423c03226446f5d84993a93 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 25 Aug 2026 23:05:38 +0300 Subject: [PATCH 14/15] HIVE-29834: Decide SMB feasibility before sizing, and take a table's size from its handler Electing the big table prices every candidate, and for a table whose parameters hold no totalSize the selector fell back to listing the table's location, which takes minutes per table on object storage. Neither cost is owed: a storage handler serves the size from its own metadata, and a join whose traits can never satisfy an SMB does not need a big table at all. The structural check consults operator traits already in memory, so it now runs first, and handler tables never reach it: their scans carry no sort or bucket traits. --- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 62 ++++++------- .../SizeBasedBigTableSelectorForAutoSMJ.java | 25 ++++++ ...stSizeBasedBigTableSelectorForAutoSMJ.java | 86 +++++++++++++++++++ 3 files changed, 142 insertions(+), 31 deletions(-) create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestSizeBasedBigTableSelectorForAutoSMJ.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 5ac61d2ec22d..f6a6ff49a2ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -476,6 +476,24 @@ private Object checkAndConvertSMBJoin(OptimizeTezProcContext context, JoinOperat fallbackToReduceSideJoin(joinOp, context); return null; } + JoinDesc joinDesc = joinOp.getConf(); + JoinCondDesc[] joinCondns = joinDesc.getConds(); + + Set joinCandidates = MapJoinProcessor.getBigTableCandidates(joinCondns); + if (joinCandidates.isEmpty()) { + // This is a full outer join. This can never be a map-join + // of any type. So return false. + return false; + } + // establish the join's structural feasibility before electing a big table: + // sizing the candidates can list their tables' locations on remote storage + if (!checkJoinSMBFeasibility(joinOp, context)) { + // we are just converting to a common merge join operator. The shuffle + // join in map-reduce case. + fallbackToReduceSideJoin(joinOp, context); + return null; + } + Class bigTableMatcherClass = null; try { String selector = HiveConf.getVar(context.parseContext.getConf(), @@ -488,14 +506,6 @@ private Object checkAndConvertSMBJoin(OptimizeTezProcContext context, JoinOperat BigTableSelectorForAutoSMJ bigTableMatcher = ReflectionUtils.newInstance(bigTableMatcherClass, null); - JoinDesc joinDesc = joinOp.getConf(); - JoinCondDesc[] joinCondns = joinDesc.getConds(); - Set joinCandidates = MapJoinProcessor.getBigTableCandidates(joinCondns); - if (joinCandidates.isEmpty()) { - // This is a full outer join. This can never be a map-join - // of any type. So return false. - return false; - } int mapJoinConversionPos = bigTableMatcher.getBigTablePosition(context.parseContext, joinOp, joinCandidates); if (mapJoinConversionPos < 0) { @@ -506,14 +516,15 @@ private Object checkAndConvertSMBJoin(OptimizeTezProcContext context, JoinOperat return null; } - if (checkConvertJoinSMBJoin(joinOp, context, mapJoinConversionPos, tezBucketJoinProcCtx)) { - convertJoinSMBJoin(joinOp, context, mapJoinConversionPos, - tezBucketJoinProcCtx.getNumBuckets(), true); - } else { - // we are just converting to a common merge join operator. The shuffle - // join in map-reduce case. - fallbackToReduceSideJoin(joinOp, context); + ReduceSinkOperator bigTableRS = + (ReduceSinkOperator) joinOp.getParentOperators().get(mapJoinConversionPos); + int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets(); + + if (numBuckets < 0) { + numBuckets = bigTableRS.getConf().getNumReducers(); } + tezBucketJoinProcCtx.setNumBuckets(numBuckets); + convertJoinSMBJoin(joinOp, context, mapJoinConversionPos, numBuckets, true); return null; } @@ -803,18 +814,12 @@ private void preserveOperatorInfos(Operator newOp, Operator oldOp, Optimiz } /* - * This method tries to convert a join to an SMB. This is done based on - * traits. If the sorted by columns are the same as the join columns then, we - * can convert the join to an SMB. Otherwise retain the bucket map join as it - * is still more efficient than a regular join. + * Decides from traits alone whether a join can become an SMB: the sorted by + * columns have to be the same as the join columns on every side. The answer + * does not depend on which side is elected the big table, so it is safe to + * consult before the election. */ - private boolean checkConvertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext context, - int bigTablePosition, TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException { - - ReduceSinkOperator bigTableRS = - (ReduceSinkOperator) joinOp.getParentOperators().get(bigTablePosition); - int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets(); - + private boolean checkJoinSMBFeasibility(JoinOperator joinOp, OptimizeTezProcContext context) { int size = -1; boolean shouldCheckExternalTables = context.conf.getBoolVar(HiveConf.ConfVars.HIVE_DISABLE_UNSAFE_EXTERNALTABLE_OPERATIONS); @@ -908,11 +913,6 @@ private boolean checkConvertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcCont } } - if (numBuckets < 0) { - numBuckets = bigTableRS.getConf().getNumReducers(); - } - tezBucketJoinProcCtx.setNumBuckets(numBuckets); - // With bucketing using two different versions. Version 1 for exiting // tables and version 2 for new tables. All the inputs to the SMB must be // from same version. This only applies to tables read directly and not diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java index a0af28976ca6..1c5186429afd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java @@ -23,6 +23,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.util.Map; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -73,15 +75,38 @@ private long getSize(HiveConf conf, String size, Path path) { } protected long getSize(HiveConf conf, Table table) { + // a storage handler serves the size from its own metadata; the table's parameters do not + // hold it, and listing the table's location can take minutes on object storage + if (table.isNonNative()) { + return handlerSize(table); + } Path path = table.getPath(); String size = table.getProperty("totalSize"); return getSize(conf, size, path); } protected long getSize(HiveConf conf, Partition partition) { + if (partition.getTable().isNonNative()) { + return handlerSize(partition.getTable()); + } Path path = partition.getDataLocation(); String size = partition.getParameters().get("totalSize"); return getSize(conf, size, path); } + + private static long handlerSize(Table table) { + if (table.getStorageHandler() != null && table.getStorageHandler().canProvideBasicStatistics()) { + Map stats = table.getStorageHandler().getBasicStatistics(table); + String size = stats != null ? stats.get(StatsSetupConst.TOTAL_SIZE) : null; + if (size != null) { + try { + return Long.parseLong(size); + } catch (NumberFormatException e) { + return -1; + } + } + } + return -1; + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestSizeBasedBigTableSelectorForAutoSMJ.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestSizeBasedBigTableSelectorForAutoSMJ.java new file mode 100644 index 000000000000..2d407233f6c6 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestSizeBasedBigTableSelectorForAutoSMJ.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.Map; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * The big-table choice for an automatic sort-merge join must take a handler table's size from + * the handler's statistics, and must never list the table's location for one. + */ +public class TestSizeBasedBigTableSelectorForAutoSMJ { + + private final SizeBasedBigTableSelectorForAutoSMJ selector = new TableSizeBasedBigTableSelectorForAutoSMJ(); + private final HiveConf conf = new HiveConf(); + + private Table handlerTable(Map basicStats) { + HiveStorageHandler handler = Mockito.mock(HiveStorageHandler.class); + Mockito.when(handler.canProvideBasicStatistics()).thenReturn(true); + Mockito.when(handler.getBasicStatistics(Mockito.any())).thenReturn(basicStats); + Table table = Mockito.mock(Table.class); + Mockito.when(table.isNonNative()).thenReturn(true); + Mockito.when(table.getStorageHandler()).thenReturn(handler); + return table; + } + + @Test + public void handlerTableSizeComesFromItsStatisticsWithoutTouchingTheFilesystem() { + Table table = handlerTable(Map.of(StatsSetupConst.TOTAL_SIZE, "12345")); + + Assert.assertEquals(12345, selector.getSize(conf, table)); + // the location is only asked for on the listing fallback, which a handler table never takes + Mockito.verify(table, Mockito.never()).getPath(); + } + + @Test + public void handlerTableOfUnknownSizeReportsUnknownRatherThanListing() { + Table table = handlerTable(Map.of()); + + Assert.assertEquals(-1, selector.getSize(conf, table)); + Mockito.verify(table, Mockito.never()).getPath(); + } + + @Test + public void handlerPartitionSizeComesFromTheTableStatistics() { + Table table = handlerTable(Map.of(StatsSetupConst.TOTAL_SIZE, "777")); + Partition partition = Mockito.mock(Partition.class); + Mockito.when(partition.getTable()).thenReturn(table); + + Assert.assertEquals(777, selector.getSize(conf, partition)); + Mockito.verify(partition, Mockito.never()).getDataLocation(); + } + + @Test + public void nativeTableKeepsReadingItsParameter() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.isNonNative()).thenReturn(false); + Mockito.when(table.getProperty("totalSize")).thenReturn("4242"); + + Assert.assertEquals(4242, selector.getSize(conf, table)); + } +} From 49fdeac0e59e75b6b50f18551cfe6bf862e54030 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 25 Aug 2026 23:27:34 +0300 Subject: [PATCH 15/15] HIVE-29834: Answer only from statistics that describe the asked partitions, columns and snapshot A column the file holds no entry for was never measured - the footer names the measured columns, a table-level file on each blob and a partition-level file on its first, so a column the schema gained since the write, which moved no snapshot, is refused at either granularity. A scan qualified by a branch, a tag, a point in time or a metadata table is not described by the metastore's single row, and its row count is answered from the snapshot the scan resolves. --- .../mr/hive/HiveIcebergStorageHandler.java | 19 +++++++---- .../iceberg/mr/hive/IcebergTableUtil.java | 19 +++++++++++ .../mr/hive/TestHiveIcebergStatistics.java | 34 ++++++++++++++++++- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 323cf0955097..b380c6f00673 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -227,7 +227,6 @@ public class HiveIcebergStorageHandler extends DefaultStorageHandler implements private static final String TABLE_NAME_SEPARATOR = ".."; public static final String TABLE_DEFAULT_LOCATION = "TABLE_DEFAULT_LOCATION"; - private static final String PARTITION = IcebergTableUtil.PARTITION_FIELD; private static final String PARTITION_STATS_PREFIX = "partitionStats."; public static final String COPY_ON_WRITE = RowLevelOperationMode.COPY_ON_WRITE.modeName(); @@ -726,10 +725,11 @@ public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsT @Override public boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, String colName) { if (canSetColStatistics(hmsTable)) { - return IcebergTableUtil.colStatsAccurate(hmsTable, conf); + return IcebergTableUtil.colStatsAccurate(hmsTable, colName, conf); } - // the metastore holds them, and its single row describes the table: a branch has none - return hmsTable.getSnapshotRef() == null && + // the metastore holds them, and its single row describes the current table: a scan of a + // branch, a tag, a point in time or a metadata table is not described by it + return hmsTable.getQualifier().isEmpty() && StatsSetupConst.areColumnStatsUptoDate(hmsTable.getParameters(), colName); } @@ -815,7 +815,8 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms Predicate upToDate = IcebergTableUtil.upToDateColStats(table, snapshot, statsFile, conf, true); Map> statsByPart = IcebergColStatsReader.readPartColStats(table, statsFile, - partition -> partitions.contains(partition) && upToDate.test(partition), Sets.newHashSet(colNames)); + partition -> partitions.contains(partition) && upToDate.test(partition), + columnFilter(table, colNames)); List partStats = Lists.newArrayList(); statsByPart.forEach((partition, statsObjs) -> { @@ -837,12 +838,18 @@ public AggrStats getAggrColStatsFor(org.apache.hadoop.hive.ql.metadata.Table hms return new AggrStats(colStatsList, partStats.size()); } + /** The asked columns, or null when they cover the schema: a full ask reads each blob whole. */ + private static Set columnFilter(Table table, List colNames) { + Set columns = Sets.newHashSet(colNames); + return columns.size() >= table.schema().columns().size() ? null : columns; + } + @Override public Long getRowCount(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { if (hmsTable.getMetaTable() != null) { return null; } - return getStatsSource().equals(HiveMetaHook.ICEBERG) || hmsTable.getSnapshotRef() != null ? + return getStatsSource().equals(HiveMetaHook.ICEBERG) || !hmsTable.getQualifier().isEmpty() ? snapshotRowCount(hmsTable) : metastoreRowCount(hmsTable); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 3315d4080514..2b31a16e583c 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -900,6 +900,25 @@ public static TransformSpec getTransformSpec(Table table, String transformName, return spec; } + /** + * Whether the stored column statistics answer for the column: they still describe the snapshot + * the table names, and their file holds an entry for it. The footer names the measured columns - + * a table-level file on each blob, a partition-level file on its first, so a column the schema + * gained since the write, which moved no snapshot, is refused either way. + */ + public static boolean colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, String colName, + Configuration conf) { + Table table = getTable(conf, hmsTable.getTTable()); + Snapshot snapshot = getTableSnapshot(table, hmsTable); + if (snapshot == null) { + return false; + } + StatisticsFile statsFile = getColStatsFile(table, snapshot.snapshotId(), isPartitionStats(table, conf)); + Types.NestedField field = table.schema().caseInsensitiveFindField(colName); + + return statsFile != null && field != null && + statsFile.blobMetadata().stream().anyMatch(metadata -> metadata.fields().contains(field.fieldId())); + } /** * Whether the stored column statistics still describe the table: the current snapshot owns them, diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index a1e52181897f..9fda458a2fe6 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -1867,7 +1867,13 @@ public void testMergeCompletesOnlyTheColumnsTheStoredFileDescribes() { shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 5)"); - // the file holds no half-truth for v: the increment's entry was not promoted + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + Assert.assertTrue("the analyzed column, completed by the increment, still answers", + handler.areColumnStatsUptoDate(hmsTable, "id")); + Assert.assertFalse("a column the stored file never described must not answer", + handler.areColumnStatsUptoDate(hmsTable, "v")); + // the file itself holds no half-truth for v: the increment's entry was not promoted List stored = readCurrentColStats(identifier).getFirst().getStatsObj(); Assert.assertEquals(List.of("id"), stored.stream().map(ColumnStatisticsObj::getColName).toList()); } @@ -1906,6 +1912,32 @@ public ColumnStatistics next() { Assert.assertFalse("a failed gather must not publish statistics", hasColStatsForCurrentSnapshot(identifier)); } + @Test + public void testTimeTravelIsNeverAnsweredFromTheMetastoreRow() { + // the metastore's single row describes the current snapshot; a scan of an older one must + // read its own snapshot whatever the statistics source says + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_time_travel_stats"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + long oldSnapshot = testTables.loadTable(identifier).currentSnapshot().snapshotId(); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (7), (9), (11)"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE.varname, "metastore"); + try { + org.apache.hadoop.hive.ql.metadata.Table asOf = hmsTable(identifier); + asOf.setAsOfVersion(String.valueOf(oldSnapshot)); + Assert.assertEquals("the scan reads two rows, however the table now holds five", + Long.valueOf(2), storageHandler().getRowCount(asOf)); + Assert.assertFalse("the metastore's row must not answer for a point in time", + storageHandler().areColumnStatsUptoDate(asOf, "id")); + } finally { + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE.varname, "iceberg"); + } + } + @Test public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { // the same insert onto a table that carries no statistics: an increment gathered over no rows