From 987043a2a61bd460dd7504d9a95230c48f0fbf12 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 20 Aug 2026 16:45:24 +0300 Subject: [PATCH 1/2] HIVE-29781: Introduce PARTITION__NAME virtual column and refactor Iceberg virtual column plumbing Adds VirtualColumn.PARTITION_NAME, materialized by the Iceberg readers: vectorized via HiveBatchIterator, row mode via PartitionInfo -> IOContext -> MapOperator. Refactors the read path to take everything the file already carries from the scan task instead of the per-record constants map: spec id, partition hash, file path, partition name, first row id and file sequence number are computed once per task in both VirtualColumnAwareIterator and HiveBatchIterator. Drops the now unread _partition column from the ACID read schema, along with the write-only VIRTUAL_COLS_TO_META_COLS map and the per-record helpers it fed. Fixes VectorizedParquetRecordReader taking a column type from the job column list while indexing it with a batch column number: the two orders diverge past the data columns because the batch drops virtual columns the readers cannot fill. --- .../iceberg/mr/hive/IcebergAcidUtil.java | 93 ++++++++----------- .../iceberg/mr/hive/IcebergTableUtil.java | 6 +- .../mr/hive/vector/HiveBatchIterator.java | 60 ++++++------ .../mr/hive/vector/HiveDeleteFilter.java | 2 +- .../mr/hive/vector/HiveVectorizedReader.java | 8 +- .../AbstractIcebergRecordReader.java | 2 +- .../mapreduce/IcebergMergeRecordReader.java | 5 +- .../mr/mapreduce/IcebergRecordReader.java | 24 ++--- .../positive/iceberg_explain_formatted.q.out | 6 ++ .../positive/iceberg_merge_schema.q.out | 10 +- .../llap/vectorized_iceberg_read_mixed.q.out | 14 +-- .../llap/vectorized_iceberg_read_orc.q.out | 14 +-- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../vectorized_iceberg_merge_mixed.q.out | 60 ++++++------ .../vectorized_iceberg_read_mixed.q.out | 14 +-- .../vectorized_iceberg_read_orc.q.out | 14 +-- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../hadoop/hive/ql/exec/MapOperator.java | 6 ++ .../apache/hadoop/hive/ql/io/IOContext.java | 15 ++- .../vector/VectorizedParquetRecordReader.java | 6 +- .../apache/hadoop/hive/ql/metadata/Table.java | 15 ++- .../hive/ql/metadata/VirtualColumn.java | 4 +- .../ql/optimizer/physical/Vectorizer.java | 1 + 23 files changed, 210 insertions(+), 185 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java index c615c1d82c29..0c540e6f01f1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java @@ -27,13 +27,14 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.io.PositionDeleteInfo; import org.apache.hadoop.hive.ql.io.RowLineageInfo; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.iceberg.ContentFile; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; @@ -51,16 +52,13 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.SerializationUtil; -import org.apache.iceberg.util.StructProjection; public class IcebergAcidUtil { private IcebergAcidUtil() { } - private static final Types.NestedField PARTITION_STRUCT_META_COL = null; // placeholder value in the map private static final Map FILE_READ_META_COLS = Maps.newLinkedHashMap(); - private static final Map VIRTUAL_COLS_TO_META_COLS = Maps.newLinkedHashMap(); public static final String META_TABLE_PROPERTY = "metaTable"; private static final Map DELETE_FILE_META_COLS = Maps.newLinkedHashMap(); public static final Integer PARTITION_PROJECTION_COLUMN_ID = Integer.MAX_VALUE - 6; @@ -71,17 +69,8 @@ private IcebergAcidUtil() { DELETE_FILE_META_COLS.put(MetadataColumns.ROW_POSITION, 1); FILE_READ_META_COLS.put(MetadataColumns.SPEC_ID, 0); - FILE_READ_META_COLS.put(PARTITION_STRUCT_META_COL, 1); - FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 2); - FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_SPEC_ID.getName(), MetadataColumns.SPEC_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.PARTITION_HASH.getName(), PARTITION_STRUCT_META_COL); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.FILE_PATH.getName(), MetadataColumns.FILE_PATH); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_POSITION.getName(), MetadataColumns.ROW_POSITION); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.ROW_LINEAGE_ID.getName(), MetadataColumns.ROW_ID); - VIRTUAL_COLS_TO_META_COLS.put(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER.getName(), - MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER); + FILE_READ_META_COLS.put(MetadataColumns.FILE_PATH, 1); + FILE_READ_META_COLS.put(MetadataColumns.ROW_POSITION, 2); } private static final Types.NestedField PARTITION_HASH_META_COL = Types.NestedField.required( @@ -101,18 +90,11 @@ private IcebergAcidUtil() { /** * @param dataCols The columns of the original file read schema - * @param table The table object - it is used for populating the partition struct meta column * @return The schema for reading files, extended with metadata columns */ - public static Schema createFileReadSchemaWithVirtualColums(List dataCols, Table table) { + public static Schema createFileReadSchemaWithVirtualColums(List dataCols) { List cols = Lists.newArrayListWithCapacity(dataCols.size() + FILE_READ_META_COLS.size()); - FILE_READ_META_COLS.forEach((metaCol, index) -> { - if (metaCol == PARTITION_STRUCT_META_COL) { - cols.add(MetadataColumns.metadataColumn(table, MetadataColumns.PARTITION_COLUMN_NAME)); - } else { - cols.add(metaCol); - } - }); + FILE_READ_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); cols.addAll(dataCols); return new Schema(cols); } @@ -169,12 +151,6 @@ public static int parseSpecId(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.SPEC_ID), Integer.class); } - public static long computePartitionHash(Record rec) { - StructProjection part = rec.get(FILE_READ_META_COLS.get(PARTITION_STRUCT_META_COL), StructProjection.class); - // we need to compute a hash value for the partition struct so that it can be used as a sorting key - return computeHash(part); - } - public static PartitionKey parsePartitionKey(Record rec) { String serializedStr = rec.get(SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); return SerializationUtil.deserializeFromBase64(serializedStr); @@ -190,15 +166,11 @@ public static String getSerializedPartitionKey(StructLike structLike, PartitionS return SerializationUtil.serializeToBase64(partitionKey); } - public static String parseFilePath(Record rec) { - return rec.get(FILE_READ_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - } - public static String getFilePath(Record rec) { return rec.get(DELETE_FILE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); } - public static long parseFilePosition(Record rec) { + public static long getFilePosition(Record rec) { return rec.get(FILE_READ_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); } @@ -261,12 +233,23 @@ public static class VirtualColumnAwareIterator implements CloseableIterator currentIterator, Schema expectedSchema, Configuration conf) { + private final int specId; + private final long partitionHash; + private final String filePath; + + public VirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, + Configuration conf, FileScanTask task) { this.currentIterator = currentIterator; - this.current = GenericRecord.create( - new Schema(expectedSchema.columns().subList(4, expectedSchema.columns().size()))); + this.current = GenericRecord.create(new Schema( + expectedSchema.columns().subList(FILE_READ_META_COLS.size(), expectedSchema.columns().size()))); this.conf = conf; + + this.specId = task.file().specId(); + this.partitionHash = computeHash(task.file().partition()); + this.filePath = task.file().location(); + + IOContextMap.get(conf).setPartitionName( + IcebergTableUtil.toPartitionName(task.spec(), task.file().partition())); } @Override @@ -285,10 +268,10 @@ public T next() { GenericRecord rec = (GenericRecord) next; IcebergAcidUtil.copyFields(rec, FILE_READ_META_COLS.size(), current.size(), current); PositionDeleteInfo.setIntoConf(conf, - IcebergAcidUtil.parseSpecId(rec), - IcebergAcidUtil.computePartitionHash(rec), - IcebergAcidUtil.parseFilePath(rec), - IcebergAcidUtil.parseFilePosition(rec), + specId, + partitionHash, + filePath, + IcebergAcidUtil.getFilePosition(rec), StringUtils.EMPTY); RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec), RowLineageReader.readLastUpdatedSequenceNumber(rec), conf); @@ -300,16 +283,20 @@ public static class MergeTaskVirtualColumnAwareIterator implements CloseableI private final CloseableIterator currentIterator; private final GenericRecordBuilder recordBuilder; - private final PartitionSpec partitionSpec; - private final StructLike partition; - public MergeTaskVirtualColumnAwareIterator( - CloseableIterator currentIterator, Schema expectedSchema, ContentFile contentFile, Table table) { + private final int specId; + private final long partitionHash; + private final String serializedPartitionKey; + + public MergeTaskVirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, + PartitionSpec spec, ContentFile file) { this.currentIterator = currentIterator; - this.partition = contentFile.partition(); this.recordBuilder = new GenericRecordBuilder<>( new Schema(expectedSchema.columns().subList(0, expectedSchema.columns().size()))); - this.partitionSpec = table.specs().get(contentFile.specId()); + + this.specId = spec.specId(); + this.partitionHash = computeHash(file.partition()); + this.serializedPartitionKey = getSerializedPartitionKey(file.partition(), spec); } @Override @@ -326,11 +313,13 @@ public boolean hasNext() { public T next() { T next = currentIterator.next(); GenericRecord rec = (GenericRecord) next; - return recordBuilder.withSpecId(partitionSpec.specId()) - .withPartitionHash(computeHash(partition)) + + return recordBuilder.withSpecId(specId) + .withPartitionHash(partitionHash) .withFilePath(IcebergAcidUtil.getFilePath(rec)) .withFilePosition(IcebergAcidUtil.getDeleteFilePosition(rec)) - .withPartitionKey(getSerializedPartitionKey(partition, partitionSpec)).build(); + .withPartitionKey(serializedPartitionKey) + .build(); } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 897771bbb3cc..728c69f98f7d 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -493,8 +493,10 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy * name an unpartitioned spec renders. Statistics and partition pruning join on this name, so both must * render it the same way. */ - static String toPartitionName(PartitionSpec spec, PartitionData data) { - return StringUtils.defaultIfEmpty(spec.partitionToPath(data), DummyPartition.VOID); + public static String toPartitionName(PartitionSpec spec, StructLike data) { + String path = spec.partitionToPath(data); + // an unpartitioned spec renders nothing: its rows belong to the table-level partition + return path.isEmpty() ? DummyPartition.VOID : path; } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java index 68a910a950ae..aa2c2be78b4a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Map; import java.util.stream.LongStream; import org.apache.hadoop.hive.llap.LlapHiveUtils; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; @@ -34,10 +33,10 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.mr.hive.IcebergAcidUtil; -import org.apache.iceberg.util.StructProjection; +import org.apache.iceberg.mr.hive.IcebergTableUtil; /** * Iterator wrapper around Hive's VectorizedRowBatch producer (MRv1 implementing) record readers. @@ -52,17 +51,32 @@ public final class HiveBatchIterator implements CloseableIterator idToConstant; + + private final int specId; + private final long partitionHash; + private final String filePath; + private final String partitionName; + + private final Long firstRowId; + private final Long fileSequenceNumber; HiveBatchIterator(RecordReader recordReader, JobConf job, - int[] partitionColIndices, Object[] partitionValues, Map idToConstant) { + int[] partitionColIndices, Object[] partitionValues, FileScanTask task) { this.recordReader = recordReader; this.key = recordReader.createKey(); this.batch = recordReader.createValue(); this.vrbCtx = LlapHiveUtils.findMapWork(job).getVectorizedRowBatchCtx(); this.partitionColIndices = partitionColIndices; this.partitionValues = partitionValues; - this.idToConstant = idToConstant; + + this.specId = task.file().specId(); + this.partitionHash = IcebergAcidUtil.computeHash(task.file().partition()); + this.filePath = task.file().location(); + this.partitionName = IcebergTableUtil.toPartitionName(task.spec(), task.file().partition()); + + this.firstRowId = task.file().firstRowId(); + this.fileSequenceNumber = task.file().fileSequenceNumber(); + RowLineageUtils.initializeRowLineageColumns(vrbCtx, batch); } @@ -99,24 +113,14 @@ private void advance() { int idx = vrbCtx.findVirtualColumnNum(vc); switch (vc) { case PARTITION_SPEC_ID: - value = idToConstant.get(MetadataColumns.SPEC_ID.fieldId()); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], specId, idx); break; case PARTITION_HASH: - value = IcebergAcidUtil.computeHash( - (StructProjection) idToConstant.get(MetadataColumns.PARTITION_COLUMN_ID)); - vrbCtx.addPartitionColsToBatch(batch.cols[idx], value, idx); + vrbCtx.addPartitionColsToBatch(batch.cols[idx], partitionHash, idx); break; case FILE_PATH: - value = idToConstant.get(MetadataColumns.FILE_PATH.fieldId()); BytesColumnVector bcv = (BytesColumnVector) batch.cols[idx]; - if (value == null) { - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; - } else { - bcv.fill(((String) value).getBytes()); - } + bcv.fill(filePath.getBytes()); break; case ROW_POSITION: value = LongStream.range(rowOffset, rowOffset + batch.size).toArray(); @@ -132,10 +136,13 @@ private void advance() { bcv.isNull[0] = true; bcv.isRepeating = true; break; + case PARTITION_NAME: + bcv = (BytesColumnVector) batch.cols[idx]; + bcv.fill(partitionName.getBytes()); + break; case ROW_LINEAGE_ID: LongColumnVector rowIdLcv = (LongColumnVector) batch.cols[idx]; - Object firstRowIdObj = idToConstant.get(MetadataColumns.ROW_ID.fieldId()); - if (firstRowIdObj == null) { + if (firstRowId == null) { rowIdLcv.noNulls = false; rowIdLcv.isNull[0] = true; rowIdLcv.isRepeating = true; @@ -144,13 +151,13 @@ private void advance() { // If vector[0] is still -1, the reader didn't find the column in the file. if (rowIdLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; } } else { // Lineage data was found (could be 0). Preserve it and fill only the NULL gaps. for (int i = 0; i < batch.size; i++) { if (rowIdLcv.isNull[i]) { - rowIdLcv.vector[i] = (Long) firstRowIdObj + rowOffset + i; + rowIdLcv.vector[i] = firstRowId + rowOffset + i; rowIdLcv.isNull[i] = false; } } @@ -161,8 +168,7 @@ private void advance() { case LAST_UPDATED_SEQUENCE_NUMBER: LongColumnVector lusnLcv = (LongColumnVector) batch.cols[idx]; - Object fileSeqObj = idToConstant.get(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); - if (fileSeqObj == null) { + if (fileSequenceNumber == null) { lusnLcv.noNulls = false; lusnLcv.isNull[0] = true; lusnLcv.isRepeating = true; @@ -172,13 +178,13 @@ private void advance() { // If vector[0] is still -1, apply the file-level sequence number to the whole batch. if (lusnLcv.vector[0] == -1L) { for (int i = 0; i < batch.size; i++) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; } } else { // Lineage data found in file, fill only the gaps where data is missing. for (int i = 0; i < batch.size; i++) { if (!lusnLcv.noNulls && lusnLcv.isNull[i]) { - lusnLcv.vector[i] = (Long) fileSeqObj; + lusnLcv.vector[i] = fileSequenceNumber; lusnLcv.isNull[i] = false; } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java index 73ea486a3397..2444132705d0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java @@ -88,7 +88,7 @@ public CloseableIterable filterBatch(CloseableIterable iterator = new DeleteFilterBatchIterator(batches); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java index bfc7cc1fc5ba..0bbc5aa8f08e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java @@ -184,7 +184,7 @@ protected DeleteLoader newDeleteLoader() { } CloseableIterable vrbIterable = - createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, idToConstant); + createVectorizedRowBatchIterable(recordReader, job, partitionColIndices, partitionValues, task); return deleteFilter != null ? deleteFilter.filterBatch(vrbIterable) : vrbIterable; @@ -293,12 +293,12 @@ private static RecordReader parquetRecordReade private static CloseableIterable createVectorizedRowBatchIterable( RecordReader hiveRecordReader, JobConf job, int[] partitionColIndices, - Object[] partitionValues, Map idToConstant) { + Object[] partitionValues, FileScanTask task) { HiveBatchIterator iterator = - new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, idToConstant); + new HiveBatchIterator(hiveRecordReader, job, partitionColIndices, partitionValues, task); - return new CloseableIterable() { + return new CloseableIterable<>() { @Override public CloseableIterator iterator() { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java index 29f285cf37b1..a0a51b840626 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java @@ -81,7 +81,7 @@ private static Schema readSchema(Configuration conf, Table table, boolean caseSe } if (InputFormatConfig.fetchVirtualColumns(conf)) { - readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns(), table); + readSchema = IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns()); if (IcebergTableUtil.supportsRowLineage(table.properties())) { readSchema = MetadataColumns.schemaWithRowLineage(readSchema); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java index 8c0dac35f1c1..a86f3954dbba 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java @@ -26,6 +26,7 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.avro.PlannedDataReader; @@ -58,8 +59,10 @@ private CloseableIterator nextTask() { CloseableIterator closeableIterator = openGeneric(mergeSplit.getContentFile(), table.schema()).iterator(); if (mergeSplit.getContentFile() instanceof DeleteFile) { Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns()); + PartitionSpec spec = table.specs().get(mergeSplit.getContentFile().specId()); + return new IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator, - deleteSchema, mergeSplit.getContentFile(), table); + deleteSchema, spec, mergeSplit.getContentFile()); } else { return closeableIterator; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 635a311a6c00..792d97c73f17 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -37,8 +37,6 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; @@ -86,12 +84,13 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { } private CloseableIterator nextTask() { - CloseableIterator closeableIterator = open(tasks.next(), expectedSchema).iterator(); + FileScanTask task = tasks.next(); + CloseableIterator closeableIterator = open(task, expectedSchema).iterator(); if (!isFetchVirtualColumns() || Utilities.getIsVectorized(conf)) { return closeableIterator; } return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator, - expectedSchema, conf); + expectedSchema, conf, task); } @Override @@ -262,19 +261,12 @@ private CloseableIterable newOrcIterable( } private Map constantsMap(FileScanTask task, BiFunction converter) { - PartitionSpec spec = task.spec(); - Set idColumns = spec.identitySourceIds(); - Schema partitionSchema = TypeUtil.select(expectedSchema, idColumns); - boolean projectsIdentityPartitionColumns = !partitionSchema.columns().isEmpty(); - if (expectedSchema.findField(MetadataColumns.PARTITION_COLUMN_ID) != null) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else if (projectsIdentityPartitionColumns) { - Types.StructType partitionType = Partitioning.partitionType(table); - return PartitionUtil.constantsMap(task, partitionType, converter); - } else { - return Collections.emptyMap(); + boolean projectsIdentityPartitionColumns = !TypeUtil.select(expectedSchema, task.spec().identitySourceIds()) + .columns().isEmpty(); + if (expectedSchema.findField(MetadataColumns.SPEC_ID.fieldId()) != null || projectsIdentityPartitionColumns) { + return PartitionUtil.constantsMap(task, converter); } + return Collections.emptyMap(); } private static Schema schemaWithoutConstantsAndMeta(Schema readSchema, Map idToConstant) { diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out index 0032d06a3584..2382567e0e53 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out @@ -67,6 +67,12 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### "precision": 2147483647, "name": "PARTITION__PROJECTION" }, + { + "type": "VARCHAR", + "nullable": true, + "precision": 2147483647, + "name": "PARTITION__NAME" + }, { "type": "BIGINT", "nullable": true, diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out index 4cd3f87b9aaf..45a08b468787 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -202,7 +202,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string, 11:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -210,14 +210,14 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [6, 7, 8, 9, 10, 11, 1, 2, 3, 4, 5] - selectExpressions: ConstantVectorExpression(val 1090969) -> 11:bigint + projectedOutputColumnNums: [6, 7, 8, 9, 10, 12, 1, 2, 3, 4, 5] + selectExpressions: ConstantVectorExpression(val 1090969) -> 12:bigint Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumns: 4:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 11:bigint, 1:string, 2:string, 3:int, 5:string + valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 12:bigint, 1:string, 2:string, 3:int, 5:string Filter Vectorization: className: VectorFilterOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index 37bff7329394..a5d2bdd45ad5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -235,7 +235,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -390,7 +390,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -399,7 +399,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -516,7 +516,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -916,7 +916,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index c215c4005a46..ecc0674d0785 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -139,7 +139,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -359,7 +359,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -368,7 +368,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -520,7 +520,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -655,7 +655,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -768,7 +768,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -942,7 +942,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index b2ea3e67b861..3a458a35794e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -357,7 +357,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -558,7 +558,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 6965a3604c45..9b971e5703ff 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -338,15 +338,15 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] Select Operator expressions: ss_sold_time_sk (type: int), ss_item_sk2 (type: int), ss_customer_sk2 (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)), (floor((UDFToDouble(ss_item_sk2) / 1000.0D)) * 1000L) BETWEEN 1000L AND 2000L (type: boolean), (ss_ext_discount_amt < 0) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] - selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean + projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 33, 34] + selectExpressions: LongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 30:double)(children: DoubleColDivideDoubleScalar(col 29:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 29:double) -> 30:double) -> 31:bigint) -> 32:bigint) -> 33:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 34:boolean Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -358,16 +358,16 @@ STAGE PLANS: 0 _col2 (type: int), _col1 (type: int) 1 _col8 (type: int), _col7 (type: int) Map Join Vectorization: - bigTableFilterExpressions: SelectColumnIsTrue(col 32:boolean), SelectColumnIsTrue(col 33:boolean) + bigTableFilterExpressions: SelectColumnIsTrue(col 33:boolean), SelectColumnIsTrue(col 34:boolean) bigTableKeyColumns: 3:int, 2:int bigTableRetainColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] bigTableValueColumns: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) className: VectorMapJoinOuterMultiKeyOperator native: true nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - outerSmallTableKeyMapping: 2 -> 41, 3 -> 42 - projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) - smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:string, 39:int, 40:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + outerSmallTableKeyMapping: 2 -> 42, 3 -> 43 + projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + smallTableValueMapping: 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) hashTableImplementationType: OPTIMIZED outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 input vertices: @@ -379,13 +379,13 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [50, 20, 43, 49, 34, 22, 36, 8, 4, 11, 59, 48, 38, 7, 60, 12, 44, 18, 15, 55, 57, 21, 9, 45, 41, 51, 46, 5, 58, 6, 14, 13, 54, 53, 2, 17, 39, 3, 16, 19, 35, 10, 40, 52, 1, 37, 61, 42, 47, 56] + projectedOutputColumnNums: [51, 20, 44, 50, 35, 22, 37, 8, 4, 11, 60, 49, 39, 7, 61, 12, 45, 18, 15, 56, 58, 21, 9, 46, 42, 52, 47, 5, 59, 6, 14, 13, 55, 54, 2, 17, 40, 3, 16, 19, 36, 10, 41, 53, 1, 38, 62, 43, 48, 57] Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -394,8 +394,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [34, 35, 36, 37, 38, 30, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 62, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 62:decimal(7,2) + projectedOutputColumnNums: [35, 36, 37, 38, 39, 31, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 63, 55, 56, 57, 58, 59, 60, 61, 62] + selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val null) -> 63:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -412,7 +412,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 53:decimal(7,2))) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -421,8 +421,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [31, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 63, 54, 55, 56, 57, 58, 59, 60, 61] - selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 63:decimal(7,2) + projectedOutputColumnNums: [32, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 64, 55, 56, 57, 58, 59, 60, 61, 62] + selectExpressions: ConstantVectorExpression(val 2451181) -> 32:int, ConstantVectorExpression(val 0) -> 64:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -431,18 +431,18 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 42:int, 65:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int + keyColumns: 43:int, 66:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 31:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 63:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2) + valueColumns: 32:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 64:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 39:int)) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 43:int), SelectColumnIsNull(col 40:int)) predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -451,8 +451,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 64:int + projectedOutputColumnNums: [65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 65:int Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -461,18 +461,18 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int + keyColumns: 3:int, 67:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 67:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 64:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 65:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 41:int, col 2:int), FilterLongColEqualLongColumn(col 42:int, col 3:int), FilterLongColEqualLongScalar(col 39:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator @@ -481,7 +481,7 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [34, 36, 35, 37] + projectedOutputColumnNums: [35, 37, 36, 38] Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -489,7 +489,7 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 34:int, col 35:bigint, col 36:string, col 37:bigint + keyExpressions: col 35:int, col 36:bigint, col 37:string, col 38:bigint native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -535,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] Filter Operator Filter Vectorization: className: VectorFilterOperator @@ -549,8 +549,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [23, 24, 25, 26, 27, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int + projectedOutputColumnNums: [23, 24, 25, 26, 27, 29, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 29:int Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col8 (type: int), _col7 (type: int) @@ -563,7 +563,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int - valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 29:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) Execution mode: vectorized diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index eeb9cbfe354f..e4275452b154 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -197,7 +197,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -351,7 +351,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -360,7 +360,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -476,7 +476,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -797,7 +797,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 0b7260d84206..3e4c4a1e6e0a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -102,7 +102,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -282,7 +282,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -291,7 +291,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -442,7 +442,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -576,7 +576,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -688,7 +688,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -861,7 +861,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index b4a16f6690d8..71cff814d471 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -280,7 +280,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -480,7 +480,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 221d2ac08f6f..64bd708c37ea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -686,6 +686,12 @@ public static Object[] populateVirtualColumnValues(ExecMapperContext ctx, vcValues[i] = new Text(ctx.getIoCxt().getPositionDeleteInfo().getPartitionProjection()); } break; + case PARTITION_NAME: + vcValues[i] = null; + if (ctx.getIoCxt().getPartitionName() != null) { + vcValues[i] = new Text(ctx.getIoCxt().getPartitionName()); + } + break; case FILE_PATH: vcValues[i] = null; if (ctx.getIoCxt().getPositionDeleteInfo() != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java index ad438fd1b2e2..0dd8b3afb299 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java @@ -51,12 +51,15 @@ public class IOContext { /** * supports {@link org.apache.hadoop.hive.ql.metadata.VirtualColumn#ROWID} */ - private RecordIdentifier ri; + private RecordIdentifier ri; private boolean isDeletedRecord; private PositionDeleteInfo pdi; private RowLineageInfo rowLineageInfo; - public static enum Comparison { + // the partition of the rows the reader currently serves + private String partitionName; + + public enum Comparison { GREATER, LESS, EQUAL, @@ -199,6 +202,14 @@ public PositionDeleteInfo getPositionDeleteInfo() { return pdi; } + public void setPartitionName(String partitionName) { + this.partitionName = partitionName; + } + + public String getPartitionName() { + return partitionName; + } + public void parseRowLineageInfo(JobConf cconfiguration) { this.rowLineageInfo = RowLineageInfo.parseFromConf(cconfiguration); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 03e7e5006474..236f6f3095f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -458,9 +458,11 @@ private void checkEndOfRowGroup() throws IOException { if(!colsToInclude.isEmpty()) { for (int i = 0; i < types.size(); ++i) { columnReaders[i] = - buildVectorizedParquetReader(columnTypesList.get(colsToInclude.get(i)), types.get(i), + buildVectorizedParquetReader( + columnTypesList.get(colsToInclude.get(i)), types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, 0, 0); + legacyConversionEnabled, 0, 0 + ); } } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 2592113614cc..a46e8c2cb95a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -1463,16 +1463,21 @@ public List getVirtualColumns() { List virtualColumns = new ArrayList<>(); if (!isNonNative()) { virtualColumns.addAll(VirtualColumn.getRegistry()); + return virtualColumns; } - if (isNonNative() && AcidUtils.isNonNativeAcidTable(this)) { + if (AcidUtils.isNonNativeAcidTable(this)) { virtualColumns.addAll(getStorageHandler().acidVirtualColumns()); } - if (isNonNative() && getStorageHandler().areSnapshotsSupported() && - isBlank(getMetaTable())) { + if (!isBlank(getMetaTable())) { + return virtualColumns; + } + if (hasNonNativePartitionSupport()) { + virtualColumns.add(VirtualColumn.PARTITION_NAME); + } + if (getStorageHandler().areSnapshotsSupported()) { virtualColumns.add(VirtualColumn.SNAPSHOT_ID); } - if (isNonNative() && getStorageHandler().supportsRowLineage(getTTable().getParameters()) && - isBlank(getMetaTable())) { + if (getStorageHandler().supportsRowLineage(getTTable().getParameters())) { virtualColumns.add(VirtualColumn.ROW_LINEAGE_ID); virtualColumns.add(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index 227dda27d9d2..fb5852cf144c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -58,6 +58,7 @@ public enum VirtualColumn { ROW_POSITION("ROW__POSITION", TypeInfoFactory.longTypeInfo), SNAPSHOT_ID("SNAPSHOT__ID", TypeInfoFactory.longTypeInfo), PARTITION_PROJECTION("PARTITION__PROJECTION", TypeInfoFactory.stringTypeInfo), + PARTITION_NAME("PARTITION__NAME", TypeInfoFactory.stringTypeInfo), ROW_LINEAGE_ID("ROW__LINEAGE__ID", TypeInfoFactory.longTypeInfo), LAST_UPDATED_SEQUENCE_NUMBER("LAST__UPDATED__SEQUENCE__NUMBER", TypeInfoFactory.longTypeInfo), @@ -74,7 +75,8 @@ public enum VirtualColumn { ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName(), ROWISDELETED.getName(), PARTITION_SPEC_ID.getName(), PARTITION_HASH.getName(), FILE_PATH.getName(), ROW_POSITION.getName(), - PARTITION_PROJECTION.getName(), ROW_LINEAGE_ID.getName(), LAST_UPDATED_SEQUENCE_NUMBER.getName()); + PARTITION_PROJECTION.getName(), PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), + LAST_UPDATED_SEQUENCE_NUMBER.getName()); public static final ImmutableMap VIRTUAL_COLUMN_NAME_MAP = new ImmutableMap.Builder().putAll(getColumnNameMap()).build(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 512337c11fdb..7d0a3cb2a8cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -310,6 +310,7 @@ public class Vectorizer implements PhysicalPlanResolver { VirtualColumn.FILE_PATH, VirtualColumn.ROW_POSITION, VirtualColumn.PARTITION_PROJECTION, + VirtualColumn.PARTITION_NAME, VirtualColumn.ROW_LINEAGE_ID, VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); private HiveConf hiveConf; From ced7038c5cf1a03818b683f847a346d8f4617930 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 20 Aug 2026 19:40:03 +0300 Subject: [PATCH 2/2] HIVE-29781: Make PARTITION__PROJECTION a merge task record field instead of a virtual column The partition key a merge task carries is read back only by HiveIcebergDeleteWriter under isMergeTask: an ordinary delete derives the key from the row data, and a merge task cannot because it reads delete files, whose records hold only a path and a position. Declaring it as a virtual column put an always empty string into every DELETE, UPDATE and MERGE record and into every Iceberg plan. It now lives in a merge specific serde layout that IcebergMergeRecordReader and HiveIcebergSerDe select on the merge task flag, so the ordinary delete record is one column shorter and the plans no longer estimate it. --- .../iceberg/mr/hive/HiveIcebergSerDe.java | 17 +- .../mr/hive/HiveIcebergStorageHandler.java | 7 +- .../iceberg/mr/hive/IcebergAcidUtil.java | 72 +-- .../mr/hive/vector/HiveBatchIterator.java | 6 - .../HiveIcebergCopyOnWriteRecordWriter.java | 2 +- .../hive/writer/HiveIcebergDeleteWriter.java | 2 +- .../mapreduce/IcebergMergeRecordReader.java | 2 +- .../mr/mapreduce/IcebergRecordReader.java | 2 +- .../mr/hive/TestHiveIcebergSelects.java | 17 + .../vector/TestHiveIcebergVectorization.java | 10 +- .../writer/TestHiveIcebergDeleteWriter.java | 5 +- ...te_iceberg_copy_on_write_partitioned.q.out | 286 +++++------ ..._iceberg_copy_on_write_unpartitioned.q.out | 278 +++++------ .../positive/delete_iceberg_mixed.q.out | 10 +- .../positive/iceberg_explain_formatted.q.out | 6 - .../positive/iceberg_merge_schema.q.out | 10 +- ...rg_truncate_partition_with_evolution.q.out | 40 +- .../llap/vectorized_iceberg_read_mixed.q.out | 14 +- .../llap/vectorized_iceberg_read_orc.q.out | 14 +- .../vectorized_iceberg_read_parquet.q.out | 8 +- ...ge_iceberg_copy_on_write_partitioned.q.out | 164 +++---- ..._iceberg_copy_on_write_unpartitioned.q.out | 322 ++++++------- .../results/positive/merge_iceberg_orc.q.out | 74 +-- .../merge_iceberg_partitioned_orc.q.out | 74 +-- ...merge_with_null_check_on_joining_col.q.out | 140 +++--- .../results/positive/metadata_delete.q.out | 8 +- .../results/positive/mv_iceberg_orc4.q.out | 42 +- .../results/positive/mv_iceberg_orc5.q.out | 42 +- .../results/positive/mv_iceberg_orc7.q.out | 22 +- .../results/positive/mv_iceberg_orc8.q.out | 42 +- ...te_iceberg_copy_on_write_partitioned.q.out | 452 +++++++++--------- ..._iceberg_copy_on_write_unpartitioned.q.out | 360 +++++++------- .../vectorized_iceberg_merge_mixed.q.out | 126 ++--- .../vectorized_iceberg_read_mixed.q.out | 14 +- .../vectorized_iceberg_read_orc.q.out | 14 +- .../vectorized_iceberg_read_parquet.q.out | 8 +- .../positive/write_iceberg_branch.q.out | 60 +-- .../hadoop/hive/ql/exec/MapOperator.java | 6 - .../hadoop/hive/ql/io/PositionDeleteInfo.java | 16 +- .../hive/ql/metadata/VirtualColumn.java | 3 +- .../ql/optimizer/physical/Vectorizer.java | 1 - 41 files changed, 1400 insertions(+), 1398 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java index 83d00942edd5..68dc85413742 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java @@ -140,8 +140,7 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - this.projectedSchema = - projectedSchema(conf, serDeProperties.getProperty(Catalogs.NAME), tableSchema, jobConf); + this.projectedSchema = projectedSchema(conf, serDeProperties, tableSchema, jobConf); if (!IcebergTableUtil.isFanoutEnabled(serDeProperties::getProperty)) { // ClusteredWriter requires that records are ordered by partition keys. @@ -156,9 +155,11 @@ public void initialize(Configuration conf, Properties serDeProperties, } } - private static Schema projectedSchema(Configuration conf, String tableName, Schema tableSchema, - Map jobConf) { + private static Schema projectedSchema(Configuration conf, Properties serDeProperties, + Schema tableSchema, Map jobConf) { + String tableName = serDeProperties.getProperty(Catalogs.NAME); Context.Operation operation = HiveCustomStorageHandlerUtils.getWriteOperation(conf::get, tableName); + if (operation == null) { jobConf.put(InputFormatConfig.CASE_SENSITIVE, "false"); String[] selectedColumns = ColumnProjectionUtils.getReadColumnNames(conf); @@ -180,11 +181,15 @@ private static Schema projectedSchema(Configuration conf, String tableName, Sche } boolean isCOW = IcebergTableUtil.isCopyOnWriteMode(operation, conf::get); if (isCOW) { - return getSchemaWithRowLineage(IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()), conf); + return getSchemaWithRowLineage( + IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), false), conf); } switch (operation) { case DELETE: - return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns()); + boolean isMergeTask = HiveCustomStorageHandlerUtils.isMergeTaskEnabled( + key -> serDeProperties.getProperty(key, conf.get(key)), + tableName); + return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns(), isMergeTask); case UPDATE: return IcebergAcidUtil.createSerdeSchemaForUpdate(tableSchema.columns()); case OTHER: diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b23c20991856..d79275668a67 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -214,7 +214,7 @@ import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.FILE_PATH; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_HASH; -import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_PROJECTION; +import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_NAME; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.PARTITION_SPEC_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_LINEAGE_ID; import static org.apache.hadoop.hive.ql.metadata.VirtualColumn.ROW_POSITION; @@ -244,7 +244,7 @@ public class HiveIcebergStorageHandler extends DefaultStorageHandler implements public static final String MERGE_ON_READ = RowLevelOperationMode.MERGE_ON_READ.modeName(); private static final List ACID_VIRTUAL_COLS = ImmutableList.of( - PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION, PARTITION_PROJECTION); + PARTITION_SPEC_ID, PARTITION_HASH, FILE_PATH, ROW_POSITION); private static final List ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA = schema(ACID_VIRTUAL_COLS); @@ -427,7 +427,8 @@ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer dese List subExprNodes = pushedPredicate.getChildren(); Set skipList = - Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) + Stream.of(FILE_PATH, PARTITION_SPEC_ID, PARTITION_HASH, PARTITION_NAME, + ROW_LINEAGE_ID, LAST_UPDATED_SEQUENCE_NUMBER) .map(VirtualColumn::getName).collect(Collectors.toSet()); if (subExprNodes.removeIf(nodeDesc -> nodeDesc.getCols() != null && diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java index 0c540e6f01f1..b881e07b2eb8 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java @@ -25,7 +25,6 @@ import java.util.Objects; import java.util.Optional; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.hive.ql.io.PositionDeleteInfo; @@ -76,16 +75,22 @@ private IcebergAcidUtil() { private static final Types.NestedField PARTITION_HASH_META_COL = Types.NestedField.required( MetadataColumns.PARTITION_COLUMN_ID, MetadataColumns.PARTITION_COLUMN_NAME, Types.LongType.get()); - public static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( + private static final Types.NestedField PARTITION_PROJECTION = Types.NestedField.required( PARTITION_PROJECTION_COLUMN_ID, PARTITION_PROJECTION_COLUMN_NAME, Types.StringType.get()); + private static final Map SERDE_META_COLS = Maps.newLinkedHashMap(); + // a merge task reads delete files, so its writer has no row data to derive the partition key from + private static final Map MERGE_SERDE_META_COLS = Maps.newLinkedHashMap(); + static { SERDE_META_COLS.put(MetadataColumns.SPEC_ID, 0); SERDE_META_COLS.put(PARTITION_HASH_META_COL, 1); SERDE_META_COLS.put(MetadataColumns.FILE_PATH, 2); SERDE_META_COLS.put(MetadataColumns.ROW_POSITION, 3); - SERDE_META_COLS.put(PARTITION_PROJECTION, 4); + + MERGE_SERDE_META_COLS.putAll(SERDE_META_COLS); + MERGE_SERDE_META_COLS.put(PARTITION_PROJECTION, 4); } /** @@ -101,11 +106,14 @@ public static Schema createFileReadSchemaWithVirtualColums(List dataCols) { - List cols = Lists.newArrayListWithCapacity(dataCols.size() + SERDE_META_COLS.size()); - SERDE_META_COLS.forEach((metaCol, index) -> cols.add(metaCol)); + public static Schema createSerdeSchemaForDelete(List dataCols, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; + List cols = Lists.newArrayListWithCapacity(dataCols.size() + metaCols.size()); + cols.addAll(metaCols.keySet()); cols.addAll(dataCols); return new Schema(cols); } @@ -115,14 +123,17 @@ public static Schema createSerdeSchemaForDelete(List dataCols * the field values from `rec`. * @param rec The record read by the file scan task, which contains both the metadata fields and the row data fields * @param rowData The record object to populate with the rowData fields only + * @param isMergeTask Whether the record was built by a merge task * @return The position delete object */ - public static PositionDelete getPositionDelete(Record rec, Record rowData) { + public static PositionDelete getPositionDelete(Record rec, Record rowData, boolean isMergeTask) { + Map metaCols = isMergeTask ? + MERGE_SERDE_META_COLS : SERDE_META_COLS; PositionDelete positionDelete = PositionDelete.create(); - String filePath = rec.get(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), String.class); - Long filePosition = rec.get(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), Long.class); + String filePath = rec.get(metaCols.get(MetadataColumns.FILE_PATH), String.class); + Long filePosition = rec.get(metaCols.get(MetadataColumns.ROW_POSITION), Long.class); - int dataOffset = SERDE_META_COLS.size(); // position in the rec where the actual row data begins + int dataOffset = metaCols.size(); // position in the rec where the actual row data begins for (int i = dataOffset; i < rec.size(); ++i) { rowData.set(i - dataOffset, rec.get(i)); } @@ -152,7 +163,7 @@ public static int parseSpecId(Record rec) { } public static PartitionKey parsePartitionKey(Record rec) { - String serializedStr = rec.get(SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); + String serializedStr = rec.get(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), String.class); return SerializationUtil.deserializeFromBase64(serializedStr); } @@ -237,11 +248,11 @@ public static class VirtualColumnAwareIterator implements CloseableIterator currentIterator, Schema expectedSchema, + public VirtualColumnAwareIterator(CloseableIterator currentIterator, List columns, Configuration conf, FileScanTask task) { this.currentIterator = currentIterator; - this.current = GenericRecord.create(new Schema( - expectedSchema.columns().subList(FILE_READ_META_COLS.size(), expectedSchema.columns().size()))); + this.current = GenericRecord.create( + new Schema(columns.subList(FILE_READ_META_COLS.size(), columns.size()))); this.conf = conf; this.specId = task.file().specId(); @@ -271,8 +282,7 @@ public T next() { specId, partitionHash, filePath, - IcebergAcidUtil.getFilePosition(rec), - StringUtils.EMPTY); + IcebergAcidUtil.getFilePosition(rec)); RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec), RowLineageReader.readLastUpdatedSequenceNumber(rec), conf); return (T) current; @@ -282,7 +292,7 @@ public T next() { public static class MergeTaskVirtualColumnAwareIterator implements CloseableIterator { private final CloseableIterator currentIterator; - private final GenericRecordBuilder recordBuilder; + private final MergeTaskRecordBuilder recordBuilder; private final int specId; private final long partitionHash; @@ -291,8 +301,7 @@ public static class MergeTaskVirtualColumnAwareIterator implements CloseableI public MergeTaskVirtualColumnAwareIterator(CloseableIterator currentIterator, Schema expectedSchema, PartitionSpec spec, ContentFile file) { this.currentIterator = currentIterator; - this.recordBuilder = new GenericRecordBuilder<>( - new Schema(expectedSchema.columns().subList(0, expectedSchema.columns().size()))); + this.recordBuilder = new MergeTaskRecordBuilder<>(expectedSchema); this.specId = spec.specId(); this.partitionHash = computeHash(file.partition()); @@ -323,36 +332,35 @@ public T next() { } } - private static final class GenericRecordBuilder { - + private static final class MergeTaskRecordBuilder { private final GenericRecord current; - GenericRecordBuilder(Schema schema) { + MergeTaskRecordBuilder(Schema schema) { current = GenericRecord.create(schema); } - public GenericRecordBuilder withSpecId(int specId) { - current.set(SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); + public MergeTaskRecordBuilder withSpecId(int specId) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.SPEC_ID), specId); return this; } - public GenericRecordBuilder withPartitionHash(long partitionHash) { - current.set(SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); + public MergeTaskRecordBuilder withPartitionHash(long partitionHash) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_HASH_META_COL), partitionHash); return this; } - public GenericRecordBuilder withFilePath(String filePath) { - current.set(SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); + public MergeTaskRecordBuilder withFilePath(String filePath) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.FILE_PATH), filePath); return this; } - public GenericRecordBuilder withFilePosition(long filePosition) { - current.set(SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); + public MergeTaskRecordBuilder withFilePosition(long filePosition) { + current.set(MERGE_SERDE_META_COLS.get(MetadataColumns.ROW_POSITION), filePosition); return this; } - public GenericRecordBuilder withPartitionKey(String serializedPartitionKey) { - current.set(SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); + public MergeTaskRecordBuilder withPartitionKey(String serializedPartitionKey) { + current.set(MERGE_SERDE_META_COLS.get(PARTITION_PROJECTION), serializedPartitionKey); return this; } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java index aa2c2be78b4a..ef073f07a458 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveBatchIterator.java @@ -130,12 +130,6 @@ private void advance() { lcv.isRepeating = false; System.arraycopy(value, 0, lcv.vector, 0, batch.size); break; - case PARTITION_PROJECTION: - bcv = (BytesColumnVector) batch.cols[idx]; - bcv.noNulls = false; - bcv.isNull[0] = true; - bcv.isRepeating = true; - break; case PARTITION_NAME: bcv = (BytesColumnVector) batch.cols[idx]; bcv.fill(partitionName.getBytes()); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java index 4fe3b533755a..cdb03e1a89c5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergCopyOnWriteRecordWriter.java @@ -54,7 +54,7 @@ class HiveIcebergCopyOnWriteRecordWriter extends HiveIcebergDefaultWriter { @Override public void write(Writable row) throws IOException { Record record = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(record, rowDataTemplate, false); Record rowData = positionDelete.row(); if (positionDelete.pos() < 0) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java index 9365f9834ab7..6970556fc46a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDeleteWriter.java @@ -58,7 +58,7 @@ class HiveIcebergDeleteWriter extends HiveIcebergWriterBase { @Override public void write(Writable row) throws IOException { Record rec = ((Container) row).get(); - PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate); + PositionDelete positionDelete = IcebergAcidUtil.getPositionDelete(rec, rowDataTemplate, isMergeTask); int specId = IcebergAcidUtil.parseSpecId(rec); PartitionKey partitionKey = isMergeTask ? IcebergAcidUtil.parsePartitionKey(rec) : partition(positionDelete.row(), specId); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java index a86f3954dbba..824acf20e33f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java @@ -58,7 +58,7 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { private CloseableIterator nextTask() { CloseableIterator closeableIterator = openGeneric(mergeSplit.getContentFile(), table.schema()).iterator(); if (mergeSplit.getContentFile() instanceof DeleteFile) { - Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns()); + Schema deleteSchema = IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns(), true); PartitionSpec spec = table.specs().get(mergeSplit.getContentFile().specId()); return new IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator, diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 792d97c73f17..14430fdc3af2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -90,7 +90,7 @@ private CloseableIterator nextTask() { return closeableIterator; } return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator, - expectedSchema, conf, task); + expectedSchema.columns(), conf, task); } @Override diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java index f82a2f449ade..b1746c21b38e 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSelects.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; @@ -285,4 +286,20 @@ public void testHistory() throws IOException, InterruptedException { Assert.assertEquals(table.history().get(i).snapshotId(), history.get(i)[0]); } } + + @Test + public void testFilterOnPartitionName() { + // PARTITION__NAME is materialized by the reader and is not a field of the Iceberg schema, so a + // predicate on it must not be pushed down to Iceberg + TableIdentifier identifier = TableIdentifier.of("default", "part_name_filter"); + testTables.createTable(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).identity("last_name").build(), + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); + + List rows = shell.executeStatement( + "SELECT customer_id FROM " + identifier.name() + + " WHERE PARTITION__NAME = 'last_name=Brown' AND customer_id = 0"); + Assert.assertEquals(1, rows.size()); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java index 2a8b0bdc6ed1..28487b715ff4 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveIcebergVectorization.java @@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -109,9 +110,10 @@ public void testRowIterator() throws Exception { List records = TestHelper.generateRandomRecords(allSchema, 10, 0L); Table table = testTables.createTable(shell, "temptable", allSchema, fileFormat, records); - // Identify data file location - expected to be 1 file exactly - Path dataFilePath = new Path(Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) - .files().iterator()).get(0).file().path().toString()); + // Identify the scan task - expected to be 1 file exactly + FileScanTask fileScanTask = Lists.newArrayList(Lists.newArrayList(table.newScan().planTasks().iterator()).get(0) + .files().iterator()).get(0); + Path dataFilePath = new Path(fileScanTask.file().path().toString()); // Generate a mock vectorized read job JobConf jobConf = prepareMockJob(allSchema, dataFilePath); @@ -122,7 +124,7 @@ public void testRowIterator() throws Exception { inputFormat.getRecordReader(new FileSplit(dataFilePath, 0L, Long.MAX_VALUE, new String[]{}), jobConf, new MockReporter()); HiveBatchIterator hiveBatchIterator = new HiveBatchIterator( - internalVectorizedRecordReader, jobConf, null, null, null); + internalVectorizedRecordReader, jobConf, null, null, fileScanTask); // Expected to be one batch exactly HiveBatchContext hiveBatchContext = hiveBatchIterator.next(); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java index 36ff67cb2068..1f5e8cfa7a04 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/writer/TestHiveIcebergDeleteWriter.java @@ -88,7 +88,8 @@ private static List deleteRecords(Table table, Set idsTo continue; } - GenericRecord deleteRecord = GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns())); + GenericRecord deleteRecord = + GenericRecord.create(IcebergAcidUtil.createSerdeSchemaForDelete(SCHEMA.columns(), false)); int specId = (Integer) record.getField(MetadataColumns.SPEC_ID.name()); deleteRecord.setField(MetadataColumns.SPEC_ID.name(), specId); PartitionKey partitionKey = new PartitionKey(table.specs().get(specId), table.schema()); @@ -96,8 +97,6 @@ private static List deleteRecords(Table table, Set idsTo deleteRecord.setField(MetadataColumns.PARTITION_COLUMN_NAME, partitionKey); deleteRecord.setField(MetadataColumns.FILE_PATH.name(), record.getField(MetadataColumns.FILE_PATH.name())); deleteRecord.setField(MetadataColumns.ROW_POSITION.name(), record.getField(MetadataColumns.ROW_POSITION.name())); - deleteRecord.setField(IcebergAcidUtil.PARTITION_PROJECTION.name(), - IcebergAcidUtil.getSerializedPartitionKey(partitionKey, table.spec())); SCHEMA.columns().forEach(field -> deleteRecord.setField(field.name(), record.getField(field.name()))); deleteRecords.add(deleteRecord); diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out index a531c87a685c..97f2fb1f0303 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out @@ -126,16 +126,16 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2916 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1812 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -146,14 +146,14 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2868 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1764 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL @@ -358,15 +358,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2946 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1842 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -375,13 +375,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -390,15 +390,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -407,25 +407,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 18 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -444,21 +444,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 3623 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 2288 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1811 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 1144 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 19 Execution mode: vectorized Reduce Operator Tree: @@ -485,13 +485,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3207 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1993 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -618,19 +618,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 2119 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -639,13 +639,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3666 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 2330 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -654,26 +654,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3768 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 2432 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 5 Data size: 3140 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 5 Data size: 2026 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -682,29 +682,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 3454 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 2228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 5265 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 3372 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -839,8 +839,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -848,13 +848,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -862,7 +862,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1010,13 +1010,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1025,18 +1025,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1044,7 +1044,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1053,25 +1053,25 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1089,7 +1089,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1097,13 +1097,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1128,16 +1128,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 712 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1303,8 +1303,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1312,13 +1312,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1326,7 +1326,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1474,13 +1474,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1489,18 +1489,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1508,7 +1508,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1517,25 +1517,25 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1553,7 +1553,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1561,13 +1561,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1592,16 +1592,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 2 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Reduce Operator Tree: Merge Join Operator diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out index 9a5350c2e0f2..0c539335f43f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_unpartitioned.q.out @@ -54,16 +54,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -92,15 +92,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -156,9 +156,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -177,17 +177,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -301,16 +301,16 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -321,14 +321,14 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: (a <= 5) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE @@ -511,15 +511,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -528,13 +528,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -543,15 +543,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 2974 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 6 Data size: 1870 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -560,25 +560,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2998 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 6 Data size: 1894 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 17 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -597,17 +597,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 6 Data size: 2862 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1758 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 1431 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 879 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1455 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 903 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -667,13 +667,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 2926 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 6 Data size: 1822 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 20 Execution mode: vectorized Reduce Operator Tree: @@ -772,19 +772,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 6 Data size: 3022 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 6 Data size: 1918 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -793,13 +793,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 6 Data size: 3046 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 6 Data size: 1942 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -808,26 +808,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3142 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 2038 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -836,15 +836,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3395 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2107 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -993,8 +993,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1002,13 +1002,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1016,7 +1016,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1151,13 +1151,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1166,18 +1166,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1185,7 +1185,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1194,11 +1194,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 501 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1216,7 +1216,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1224,13 +1224,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 422 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1255,8 +1255,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1431,8 +1431,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1440,13 +1440,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1454,7 +1454,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1583,13 +1583,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 211 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1598,18 +1598,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1617,7 +1617,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1626,11 +1626,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 250 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1648,7 +1648,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1656,13 +1656,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1687,8 +1687,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out index aed511e4b8fe..d8f7ecf10e8d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_mixed.q.out @@ -85,14 +85,14 @@ Stage-4 File Output Operator [FS_46] table:{"name:":"default.ice01"} Select Operator [SEL_44] (rows=7 width=206) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_43] (rows=7 width=206) - Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Conds:RS_59._col4=RS_65._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_59] PartitionCols:_col4 Select Operator [SEL_56] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_53] (rows=7 width=188) predicate:((((id = 2) or (id > 4)) is null or (id < 2) or ((id > 2) and (id <= 4))) and FILE__PATH is not null) TableScan [TS_0] (rows=7 width=188) @@ -120,13 +120,13 @@ Stage-4 File Output Operator [FS_70] table:{"name:":"default.ice01"} Select Operator [SEL_69] (rows=3 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_68] (rows=3 width=188) predicate:(row_number_window_0 = 1) PTF Operator [PTF_67] (rows=7 width=188) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST","partition by:":"_col4"}] Select Operator [SEL_66] (rows=7 width=188) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] + Output:["_col0","_col1","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_58] PartitionCols:FILE__PATH diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out index 2382567e0e53..f6b8cfea653d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out @@ -61,12 +61,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### "nullable": true, "name": "ROW__POSITION" }, - { - "type": "VARCHAR", - "nullable": true, - "precision": 2147483647, - "name": "PARTITION__PROJECTION" - }, { "type": "VARCHAR", "nullable": true, diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out index 45a08b468787..4eac820ba403 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_merge_schema.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:s_key:bigint, 1:year:int, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -202,7 +202,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__PROJECTION:string, 11:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:skey:bigint, 1:hierarchy_number:string, 2:hierarchy_name:string, 3:language_id:int, 4:hierarchy_display:string, 5:orderby:string, 6:PARTITION__SPEC__ID:int, 7:PARTITION__HASH:bigint, 8:FILE__PATH:string, 9:ROW__POSITION:bigint, 10:PARTITION__NAME:string] Filter Vectorization: className: VectorFilterOperator native: true @@ -210,14 +210,14 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [6, 7, 8, 9, 10, 12, 1, 2, 3, 4, 5] - selectExpressions: ConstantVectorExpression(val 1090969) -> 12:bigint + projectedOutputColumnNums: [6, 7, 8, 9, 11, 1, 2, 3, 4, 5] + selectExpressions: ConstantVectorExpression(val 1090969) -> 11:bigint Reduce Sink Vectorization: className: VectorReduceSinkStringOperator keyColumns: 4:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 10:string, 12:bigint, 1:string, 2:string, 3:int, 5:string + valueColumns: 6:int, 7:bigint, 8:string, 9:bigint, 11:bigint, 1:string, 2:string, 3:int, 5:string Filter Vectorization: className: VectorFilterOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out index ea652fe65e98..d6a09b1d53c6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_truncate_partition_with_evolution.q.out @@ -90,12 +90,12 @@ STAGE PLANS: predicate: (a = 22) (type: boolean) Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1440 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 888 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -285,12 +285,12 @@ STAGE PLANS: predicate: (a = 226784902765739L) (type: boolean) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 226784902765739L (type: bigint), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 226784902765739L (type: bigint), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1452 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 900 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -506,12 +506,12 @@ STAGE PLANS: predicate: (b = 'ddd') (type: boolean) Statistics: Num rows: 7 Data size: 665 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), 'ddd' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), 'ddd' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 3381 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 2093 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -747,8 +747,8 @@ STAGE PLANS: predicate: (b = DATE'2022-02-07') (type: boolean) Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: bigint), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: bigint), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -935,12 +935,12 @@ STAGE PLANS: predicate: (a = 1.156748927566759E11D) (type: boolean) Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), b (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), b (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1356 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 804 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1117,8 +1117,8 @@ STAGE PLANS: predicate: ((a = 1.156748927566759E11D) and (b = DATE'2022-02-07')) (type: boolean) Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 1.156748927566759E11D (type: double), DATE'2022-02-07' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2 Data size: 128 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index a5d2bdd45ad5..51f0da923f1f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -235,7 +235,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -390,7 +390,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -399,7 +399,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -507,7 +507,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -516,7 +516,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -916,7 +916,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index ecc0674d0785..e2669a45789d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -139,7 +139,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -359,7 +359,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -368,7 +368,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -520,7 +520,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -655,7 +655,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -768,7 +768,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -942,7 +942,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index 3a458a35794e..0b66a1287bca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -126,7 +126,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -348,7 +348,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -357,7 +357,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -558,7 +558,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out index b6e253358ee6..9ed2e47d7837 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_partitioned.q.out @@ -137,16 +137,16 @@ STAGE PLANS: predicate: (a <= 100) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Execution mode: vectorized Map 12 Map Operator Tree: @@ -157,30 +157,30 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL @@ -200,9 +200,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -221,21 +221,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -243,30 +243,30 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 972 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 604 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -278,24 +278,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col5 <> _col8) or _col5 is null or _col8 is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col4 <> _col7) or _col4 is null or _col7 is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -304,15 +304,15 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 8 Data size: 3870 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 8 Data size: 2398 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -382,16 +382,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -472,16 +472,16 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -506,30 +506,30 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: _col5 is null (type: boolean) + predicate: _col4 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), _col6 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out index 115992d60112..bb4c64a103ba 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out @@ -148,41 +148,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a <= 100) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -201,24 +201,24 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Execution mode: vectorized Reducer 10 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -237,17 +237,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -259,20 +259,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 2440 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 14 Data size: 1520 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1952 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1216 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new') (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3424 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new') (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2688 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -285,16 +285,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 486 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), _col4 (type: bigint), _col5 (type: int), 'Merged' (type: string), (_col6 + 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -306,24 +306,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 is null or (_col5 = _col8)) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 is null or (_col4 = _col7)) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 6 Reduce Operator Tree: Merge Join Operator @@ -332,11 +332,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 6808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 17 Data size: 4784 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -411,16 +411,16 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 4 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 4 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 3 Vertex: Union 3 @@ -483,44 +483,44 @@ STAGE PLANS: alias: target_ice Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1568 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 832 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: (a > 100) (type: boolean) Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 3 Data size: 1425 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 3 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Filter Operator predicate: ((a > 100) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE @@ -591,20 +591,20 @@ STAGE PLANS: condition map: Full Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 14 Data size: 2420 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 14 Data size: 1500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: _col5 is null (type: boolean) - Statistics: Num rows: 10 Data size: 1936 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col4 is null (type: boolean) + Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int), concat(_col7, '-merge new 2') (type: string), null (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 10 Data size: 3412 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int), concat(_col6, '-merge new 2') (type: string), null (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 2676 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -616,24 +616,24 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean) - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 <> _col7) or (_col4 <= 100) or (_col4 is null or ((_col4 = _col7) and (_col4 > 100))) is null) (type: boolean) + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 8 Data size: 3864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 8 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -642,11 +642,11 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 598 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -659,23 +659,23 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col4 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col3 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int) - outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) + outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -694,17 +694,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: string), -1L (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 4861 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3573 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -865,41 +865,41 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: int) + key expressions: _col3 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col4 (type: int) + Map-reduce partition columns: _col3 (type: int) Statistics: Num rows: 6 Data size: 1152 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col4 (type: string), _col5 (type: int) Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: int) Filter Operator predicate: (a is not null and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -959,16 +959,16 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col5 is null or (_col5 <> _col8) or _col8 is null) (type: boolean) + predicate: (_col4 is null or (_col4 <> _col7) or _col7 is null) (type: boolean) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -976,7 +976,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -985,7 +985,7 @@ STAGE PLANS: keys: 0 _col2 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 1393 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1001,9 +1001,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col4 (type: int) + 0 _col3 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -1011,13 +1011,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 1267 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1042,8 +1042,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), -1L (type: bigint), _col3 (type: int), _col4 (type: string), _col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 633 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1059,13 +1059,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col5 (type: int) + 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), 'Merged' (type: string), (_col6 - 10) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), 'Merged' (type: string), (_col5 - 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 6 Data size: 52 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out index dcb3ecd97145..830129a81e73 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_orc.q.out @@ -99,16 +99,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -117,48 +117,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -170,10 +170,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.4 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out index 693513fa2dfc..c2529066adac 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_partitioned_orc.q.out @@ -101,16 +101,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 4 Data size: 1932 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: _col4 (type: int) + Statistics: Num rows: 4 Data size: 1196 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -119,48 +119,48 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 10 Data size: 3375 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 10 Data size: 2455 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) - Statistics: Num rows: 4 Data size: 1833 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) + Statistics: Num rows: 4 Data size: 1281 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 1449 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 897 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 483 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 299 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.target_ice Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 579 Basic stats: COMPLETE Column stats: PARTIAL + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) + Statistics: Num rows: 1 Data size: 395 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -171,10 +171,10 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: _col10 is null (type: boolean) - Statistics: Num rows: 6 Data size: 2025 Basic stats: COMPLETE Column stats: PARTIAL + predicate: _col9 is null (type: boolean) + Statistics: Num rows: 6 Data size: 1473 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator @@ -185,15 +185,15 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 576 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) Filter Operator - predicate: (_col10 = _col1) (type: boolean) - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + predicate: (_col9 = _col1) (type: boolean) + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 - Statistics: Num rows: 5 Data size: 1929 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 + Statistics: Num rows: 5 Data size: 1377 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out index 64bbd1b55dec..ca720205bf24 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/merge_with_null_check_on_joining_col.q.out @@ -71,31 +71,31 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[+($6, 100)], t__c=[$7]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[+($5, 100)], t__c=[$6]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), IS NULL($5)), >($5, 10), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), IS NOT NULL($5)), <=($5, 10), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), IS NULL($4)), >($4, 10), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), IS NOT NULL($4)), <=($4, 10), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -111,11 +111,11 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(>($0, 10), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[>($0, 10)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -141,23 +141,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(>($0, 20), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[>($0, 20)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(IS NULL($5), AND(=($5, $8), =($6, $9), >($5, 10)), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), IS NOT NULL($5)))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), IS NOT NULL($4)))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -174,12 +174,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(IS NULL($4), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(IS NULL($3), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveTableScan(table=[[default, source]], table:alias=[s]) @@ -203,21 +203,21 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[null:INTEGER], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6), IS NULL($7))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[full], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[null:INTEGER], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5), IS NULL($6))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -234,12 +234,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) @@ -264,23 +264,23 @@ POSTHOOK: Input: default@source POSTHOOK: Input: default@target POSTHOOK: Output: default@target CBO PLAN: -HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) +HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) HiveUnion(all=[true]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], a=[$7], b=[$8], c=[$9]) - HiveFilter(condition=[AND(IS NULL($5), IS NULL($6))]) - HiveJoin(condition=[AND(=($5, $7), =($6, $8))], joinType=[right], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], a=[$6], b=[$7], c=[$8]) + HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))]) + HiveJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1], c=[$2]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7]) - HiveSemiJoin(condition=[=($2, $10)], joinType=[semi]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__partition__projection=[$4], t__a=[$5], t__b=[$6], t__c=[$7], a=[$8], b=[$9]) - HiveFilter(condition=[OR(IS NULL(OR(AND(=($5, $8), =($6, $9), >($5, 10)), >($5, 20), AND(IS NULL($5), IS NULL($6)))), AND(OR(<>($5, $8), <>($6, $9), <=($5, 10)), <=($5, 20), OR(IS NOT NULL($5), IS NOT NULL($6))))]) - HiveJoin(condition=[AND(=($5, $8), =($6, $9))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) + HiveSemiJoin(condition=[=($2, $9)], joinType=[semi]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__row__position=[$3], t__a=[$4], t__b=[$5], t__c=[$6], a=[$7], b=[$8]) + HiveFilter(condition=[OR(IS NULL(OR(AND(=($4, $7), =($5, $8), >($4, 10)), >($4, 20), AND(IS NULL($4), IS NULL($5)))), AND(OR(<>($4, $7), <>($5, $8), <=($4, 10)), <=($4, 20), OR(IS NOT NULL($4), IS NOT NULL($5))))]) + HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__row__position=[$6], t__a=[$0], t__b=[$1], t__c=[$2]) HiveFilter(condition=[IS NOT NULL($5)]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) @@ -297,12 +297,12 @@ HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path= HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) HiveTableScan(table=[[default, source]], table:alias=[s]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6]) - HiveFilter(condition=[=($7, 1)]) - HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__partition__projection=[$3], t__a=[$4], t__b=[$5], t__c=[$6], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) - HiveFilter(condition=[OR(>($4, 20), AND(=($4, $7), =($5, $8), >($4, 10)))]) - HiveJoin(condition=[AND(=($4, $7), =($5, $8))], joinType=[left], algorithm=[none], cost=[not available]) - HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__partition__projection=[$7], t__a=[$0], t__b=[$1], t__c=[$2]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], _o__c3=[-1:BIGINT], t__a=[$3], t__b=[$4], t__c=[$5]) + HiveFilter(condition=[=($6, 1)]) + HiveProject(t__partition__spec__id=[$0], t__partition__hash=[$1], t__file__path=[$2], t__a=[$3], t__b=[$4], t__c=[$5], row_number_window_0=[row_number() OVER (PARTITION BY $2 ORDER BY $2 NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)]) + HiveFilter(condition=[OR(>($3, 20), AND(=($3, $6), =($4, $7), >($3, 10)))]) + HiveJoin(condition=[AND(=($3, $6), =($4, $7))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(t__partition__spec__id=[$3], t__partition__hash=[$4], t__file__path=[$5], t__a=[$0], t__b=[$1], t__c=[$2]) HiveTableScan(table=[[default, target]], table:alias=[target]) HiveProject(a=[$0], b=[$1]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) diff --git a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out index bfa7597c38e0..d37561e23c54 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/metadata_delete.q.out @@ -373,12 +373,12 @@ STAGE PLANS: predicate: (b < 5) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 792 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out index f6adb53ea76c..5fbd6803a7c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc4.q.out @@ -410,16 +410,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 984 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string) + Statistics: Num rows: 2 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: boolean), _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -474,25 +474,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 6 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 6 Data size: 1524 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col8 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col4 (type: int), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint) + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) Filter Operator predicate: _col3 (type: boolean) - Statistics: Num rows: 1 Data size: 592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -520,9 +520,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary) Filter Operator predicate: _col3 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col9 (type: string), _col10 (type: int), CASE WHEN (_col2 is null) THEN (_col11) WHEN (_col11 is null) THEN (_col2) ELSE ((_col11 + _col2)) END (type: bigint) + expressions: _col8 (type: string), _col9 (type: int), CASE WHEN (_col2 is null) THEN (_col10) WHEN (_col10 is null) THEN (_col2) ELSE ((_col10 + _col2)) END (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -552,12 +552,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 488 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out index 7fbe53177787..dabdc0989f39 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc5.q.out @@ -153,16 +153,16 @@ STAGE PLANS: alias: default.mat2 Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), _c4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE + expressions: b (type: string), c (type: int), _c2 (type: bigint), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), _c4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) null sort order: zz sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2 Data size: 1016 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col10 (type: double) + Statistics: Num rows: 2 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: double) Execution mode: vectorized Map 6 Map Operator Tree: @@ -217,25 +217,25 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: int) 1 _col0 (type: string), _col1 (type: int) nullSafes: [true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 6 Data size: 2172 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 6 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col10 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint), _col9 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: double) + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 616 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -263,9 +263,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: int), _col7 (type: bigint), _col8 (type: binary), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 4 Data size: 1448 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col11 (type: string), _col12 (type: int), CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col2) ELSE ((_col13 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col14) WHEN (_col14 is null) THEN (_col3) ELSE ((_col14 + _col3)) END) (type: double) + expressions: _col10 (type: string), _col11 (type: int), CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END (type: bigint), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint), (UDFToDouble(CASE WHEN (_col2 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col2) ELSE ((_col12 + _col2)) END) / CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4 Data size: 432 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -295,12 +295,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: int), VALUE._col7 (type: bigint), VALUE._col8 (type: bigint), VALUE._col9 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint), VALUE._col7 (type: bigint), VALUE._col8 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out index 94381e415353..a52e9ae8390f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc7.q.out @@ -120,8 +120,8 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), _c1 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -129,7 +129,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: bigint), _col2 (type: boolean), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Execution mode: vectorized Map 4 Map Operator Tree: @@ -165,25 +165,25 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) nullSafes: [true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: bigint) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: bigint) Filter Operator predicate: _col2 (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -198,7 +198,7 @@ STAGE PLANS: predicate: _col2 is null (type: boolean) Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col8 (type: int), CASE WHEN (_col1 is null) THEN (_col9) WHEN (_col9 is null) THEN (_col1) ELSE ((_col9 + _col1)) END (type: bigint) + expressions: _col7 (type: int), CASE WHEN (_col1 is null) THEN (_col8) WHEN (_col8 is null) THEN (_col1) ELSE ((_col8 + _col1)) END (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -213,8 +213,8 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out index 22ccaad5e4da..23948e8892ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc8.q.out @@ -87,16 +87,16 @@ STAGE PLANS: alias: default.mat1 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE + expressions: CAST( b AS varchar(256)) (type: varchar(256)), CAST( c AS CHAR(100)) (type: char(100)), d (type: int), _c3 (type: bigint), true (type: boolean), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) null sort order: zzz sort order: +++ Map-reduce partition columns: _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) - Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: boolean), _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -135,25 +135,25 @@ STAGE PLANS: 0 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) 1 _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int) nullSafes: [true, true, true] - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: string), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: varchar(256)), _col1 (type: char(100)), _col2 (type: int), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: varchar(256)), _col6 (type: char(100)), _col7 (type: int), _col8 (type: bigint) + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: varchar(256)), _col5 (type: char(100)), _col6 (type: int), _col7 (type: bigint) Filter Operator predicate: _col4 (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -181,9 +181,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary), _col5 (type: int), _col6 (type: struct), _col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type: int), _col11 (type: bigint), _col12 (type: binary), _col13 (type: bigint), _col14 (type: bigint), _col15 (type: bigint), _col16 (type: binary) Filter Operator predicate: _col4 is null (type: boolean) - Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: CAST( _col10 AS STRING) (type: string), CAST( _col11 AS STRING) (type: string), _col12 (type: int), CASE WHEN (_col3 is null) THEN (_col13) WHEN (_col13 is null) THEN (_col3) ELSE ((_col13 + _col3)) END (type: bigint) + expressions: CAST( _col9 AS STRING) (type: string), CAST( _col10 AS STRING) (type: string), _col11 (type: int), CASE WHEN (_col3 is null) THEN (_col12) WHEN (_col12 is null) THEN (_col3) ELSE ((_col12 + _col3)) END (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -213,12 +213,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: varchar(256)), VALUE._col6 (type: char(100)), VALUE._col7 (type: int), VALUE._col8 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: varchar(256)), VALUE._col5 (type: char(100)), VALUE._col6 (type: int), VALUE._col7 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 740 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out index 7dac902dc9ae..6f3cfd8cadb7 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out @@ -56,16 +56,16 @@ STAGE PLANS: predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 1461 Basic stats: COMPLETE Column stats: PARTIAL + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 909 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Map 4 Map Operator Tree: @@ -77,16 +77,16 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Map 6 Map Operator Tree: @@ -103,7 +103,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: PARTIAL @@ -118,12 +118,12 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -137,26 +137,26 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 472 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 7 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL PTF Operator Function definitions: Input definition @@ -175,21 +175,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3 Data size: 1428 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 3 Data size: 876 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 5 Data size: 2417 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 5 Data size: 1497 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 8 Execution mode: vectorized Reduce Operator Tree: @@ -394,28 +394,28 @@ STAGE PLANS: alias: tbl_ice Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4311 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2655 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 891 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4383 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2727 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -426,14 +426,14 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: PARTIAL @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4049 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2227 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,15 +497,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 4130 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2308 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -514,22 +514,22 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4543 Basic stats: COMPLETE Column stats: NONE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2538 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 13 Reduce Operator Tree: Merge Join Operator @@ -538,13 +538,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4821 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 14 Reduce Operator Tree: Merge Join Operator @@ -553,19 +553,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 5010 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 3188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Reducer 15 Reduce Operator Tree: Merge Join Operator @@ -574,13 +574,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 5511 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 3506 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 16 Reduce Operator Tree: Merge Join Operator @@ -589,26 +589,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 5664 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 3659 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 6 Data size: 3776 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 6 Data size: 2439 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 17 Reduce Operator Tree: Merge Join Operator @@ -617,13 +617,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4870 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 3049 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 18 Reduce Operator Tree: Merge Join Operator @@ -632,15 +632,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4951 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 3130 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -649,18 +649,18 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -705,9 +705,9 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -726,21 +726,21 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 5446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 3443 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 2420 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1530 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 21 Execution mode: vectorized Reduce Operator Tree: @@ -789,15 +789,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2772 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 24 Reduce Operator Tree: Merge Join Operator @@ -806,15 +806,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2025 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 25 Execution mode: vectorized Reduce Operator Tree: @@ -1009,29 +1009,29 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 4153 Basic stats: COMPLETE Column stats: NONE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 2682 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 9 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 6750 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1177,8 +1177,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1186,13 +1186,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1200,10 +1200,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1211,7 +1211,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1320,7 +1320,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1328,13 +1328,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1359,16 +1359,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1440,25 +1440,25 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1495,13 +1495,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -1510,18 +1510,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1529,7 +1529,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1538,19 +1538,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 @@ -1688,8 +1688,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1697,13 +1697,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1711,10 +1711,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1722,7 +1722,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1831,7 +1831,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1839,13 +1839,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1870,16 +1870,16 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1951,25 +1951,25 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Reducer 5 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: int), VALUE._col5 (type: string), VALUE._col6 (type: int), KEY.iceberg_bucket(_col4, 16) (type: int), KEY.iceberg_truncate(_col5, 3) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, iceberg_bucket(_col4, 16), iceberg_truncate(_col5, 3) File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2006,13 +2006,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2021,18 +2021,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -2040,7 +2040,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -2049,19 +2049,19 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + key expressions: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) null sort order: zz sort order: ++ - Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string) + Map-reduce partition columns: iceberg_bucket(_col4, 16) (type: int), iceberg_truncate(_col5, 3) (type: string) Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), _col6 (type: int) Union 4 Vertex: Union 4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out index 150fa60ce166..01c7c92df472 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_unpartitioned.q.out @@ -55,12 +55,12 @@ STAGE PLANS: predicate: ((a = 22) or (b) IN ('four', 'one')) (type: boolean) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), 'Changed' (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1948 Basic stats: COMPLETE Column stats: COMPLETE + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), 'Changed' (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -82,7 +82,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: FILE__PATH (type: string) Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), PARTITION__PROJECTION (type: string) + value expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) Filter Operator predicate: (((b) IN ('four', 'one') or (a = 22)) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 4 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE @@ -96,24 +96,24 @@ STAGE PLANS: predicate: ((((b) IN ('four', 'one') or (a = 22)) is null or ((b <> 'four') and (b <> 'one') and (a <> 22))) and FILE__PATH is not null) (type: boolean) Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 7 Data size: 3388 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 7 Data size: 2100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Execution mode: vectorized Reducer 4 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col7 - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -132,17 +132,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 4 Data size: 1904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1168 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 952 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -202,15 +202,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 968 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2412 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -329,28 +329,28 @@ STAGE PLANS: alias: tbl_ice Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Filter Operator predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 873 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Select Operator expressions: a (type: int), c (type: int), FILE__PATH (type: string) outputColumnNames: _col0, _col1, _col2 @@ -361,14 +361,14 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Filter Operator predicate: (c > 800) (type: boolean) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE @@ -482,13 +482,13 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13 - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12 + Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4569 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col13 (type: boolean) + Statistics: Num rows: 9 Data size: 2913 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col12 (type: boolean) Reducer 11 Reduce Operator Tree: Merge Join Operator @@ -497,26 +497,26 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col13, _col14, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12, _col13, _col14 + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean), _col14 (type: bigint), _col15 (type: bigint), _col13 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11, _col12, _col13, _col15 - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean), _col13 (type: bigint), _col14 (type: bigint), _col12 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col14 + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null) or (_col15 is not null and (_col12 <> 0L)) or ((_col2 is null or (_col13 < _col12)) and null and (_col12 <> 0L) and _col15 is null)) is null or (((_col8 = 0L) or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) and ((_col12 = 0L) or (_col15 is null and (_col13 >= _col12) and _col2 is not null)))) (type: boolean) - Statistics: Num rows: 9 Data size: 4713 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null) or (_col14 is not null and (_col11 <> 0L)) or ((_col2 is null or (_col12 < _col11)) and null and (_col11 <> 0L) and _col14 is null)) is null or (((_col7 = 0L) or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) and ((_col11 = 0L) or (_col14 is null and (_col12 >= _col11) and _col2 is not null)))) (type: boolean) + Statistics: Num rows: 9 Data size: 3057 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 12 Reduce Operator Tree: Merge Join Operator @@ -525,15 +525,15 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 6 Data size: 2910 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 6 Data size: 1806 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -661,15 +661,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 3636 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint) Reducer 19 Reduce Operator Tree: Merge Join Operator @@ -678,13 +678,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 3660 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2004 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -693,15 +693,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 4365 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint) + Statistics: Num rows: 9 Data size: 2709 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Reducer 20 Reduce Operator Tree: Merge Join Operator @@ -710,15 +710,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 9 Data size: 3732 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2076 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 21 Reduce Operator Tree: Merge Join Operator @@ -727,18 +727,18 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 3768 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 9 Data size: 4437 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed again' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 9 Data size: 2781 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -920,13 +920,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -935,15 +935,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10 - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9 + Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4461 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: boolean), _col10 (type: bigint) + Statistics: Num rows: 9 Data size: 2805 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean), _col9 (type: bigint) Reducer 5 Reduce Operator Tree: Merge Join Operator @@ -952,25 +952,25 @@ STAGE PLANS: keys: 0 _col2 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col12 - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col11 + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((_col7 <> 0L) and _col9 is not null) or ((_col10 <> 0L) and _col12 is not null)) (type: boolean) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (((_col6 <> 0L) and _col8 is not null) or ((_col9 <> 0L) and _col11 is not null)) (type: boolean) + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col5 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 9 Data size: 4497 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + Statistics: Num rows: 9 Data size: 2841 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 6 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -989,17 +989,17 @@ STAGE PLANS: window function: GenericUDAFRowNumberEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 9 Data size: 4293 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 2637 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (row_number_window_0 = 1) (type: boolean) - Statistics: Num rows: 4 Data size: 1908 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1172 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4 Data size: 1940 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4 Data size: 1204 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 19 Data size: 9287 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 5791 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat @@ -1013,13 +1013,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator null sort order: sort order: - Statistics: Num rows: 9 Data size: 4389 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + Statistics: Num rows: 9 Data size: 2733 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 9 Reduce Operator Tree: Merge Join Operator @@ -1028,19 +1028,19 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) null sort order: z sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 9 Data size: 4533 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col9 (type: bigint), _col11 (type: boolean) + Statistics: Num rows: 9 Data size: 2877 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col10 (type: boolean) Union 7 Vertex: Union 7 @@ -1160,8 +1160,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1169,13 +1169,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1183,10 +1183,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1194,7 +1194,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1342,13 +1342,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -1357,18 +1357,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1376,7 +1376,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 4 Reduce Operator Tree: Merge Join Operator @@ -1385,11 +1385,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1407,11 +1407,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1429,7 +1429,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1437,13 +1437,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 8 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1468,8 +1468,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1666,8 +1666,8 @@ STAGE PLANS: predicate: FILE__PATH is not null (type: boolean) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1675,13 +1675,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) Filter Operator predicate: a is not null (type: boolean) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1689,10 +1689,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string) + value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Select Operator - expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), PARTITION__PROJECTION (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: a (type: int), b (type: string), c (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1700,7 +1700,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string) Select Operator expressions: a (type: int) outputColumnNames: _col0 @@ -1796,7 +1796,7 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1804,13 +1804,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint) Reducer 13 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: @@ -1835,8 +1835,8 @@ STAGE PLANS: predicate: (row_number_window_0 = 1) (type: boolean) Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1917,11 +1917,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col0 (type: int), 'The last one' (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1958,13 +1958,13 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8 Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator null sort order: sort order: Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col8 (type: boolean) Reducer 7 Reduce Operator Tree: Merge Join Operator @@ -1973,18 +1973,18 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10, _col11 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col8, _col9, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col10 (type: bigint), _col11 (type: bigint), _col9 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col11 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col9 (type: bigint), _col10 (type: bigint), _col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col8 = 0L) or ((_col11 is not null and (_col8 <> 0L)) or ((_col0 is null or (_col9 < _col8)) and null and (_col8 <> 0L) and _col11 is null)) is null or (_col11 is null and (_col9 >= _col8) and _col0 is not null)) (type: boolean) + predicate: ((_col7 = 0L) or ((_col10 is not null and (_col7 <> 0L)) or ((_col0 is null or (_col8 < _col7)) and null and (_col7 <> 0L) and _col10 is null)) is null or (_col10 is null and (_col8 >= _col7) and _col0 is not null)) (type: boolean) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: string) @@ -1992,7 +1992,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint) Reducer 8 Reduce Operator Tree: Merge Join Operator @@ -2001,11 +2001,11 @@ STAGE PLANS: keys: 0 _col5 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 9 Data size: 2258 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out index 9b971e5703ff..bcb446bbfdb5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out @@ -338,15 +338,15 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk2:int, 3:ss_customer_sk2:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Select Operator expressions: ss_sold_time_sk (type: int), ss_item_sk2 (type: int), ss_customer_sk2 (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)), (floor((UDFToDouble(ss_item_sk2) / 1000.0D)) * 1000L) BETWEEN 1000L AND 2000L (type: boolean), (ss_ext_discount_amt < 0) (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 33, 34] - selectExpressions: LongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 30:double)(children: DoubleColDivideDoubleScalar(col 29:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 29:double) -> 30:double) -> 31:bigint) -> 32:bigint) -> 33:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 34:boolean + projectedOutputColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 32, 33] + selectExpressions: LongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: DoubleColDivideDoubleScalar(col 28:double, val 1000.0)(children: CastLongToDouble(col 2:int) -> 28:double) -> 29:double) -> 30:bigint) -> 31:bigint) -> 32:boolean, DecimalColLessDecimalScalar(col 14:decimal(7,2), val 0) -> 33:boolean Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -355,47 +355,47 @@ STAGE PLANS: 0 {_col22} {_col23} 1 keys: - 0 _col2 (type: int), _col1 (type: int) - 1 _col8 (type: int), _col7 (type: int) + 0 _col1 (type: int), _col2 (type: int) + 1 _col6 (type: int), _col7 (type: int) Map Join Vectorization: - bigTableFilterExpressions: SelectColumnIsTrue(col 33:boolean), SelectColumnIsTrue(col 34:boolean) - bigTableKeyColumns: 3:int, 2:int + bigTableFilterExpressions: SelectColumnIsTrue(col 32:boolean), SelectColumnIsTrue(col 33:boolean) + bigTableKeyColumns: 2:int, 3:int bigTableRetainColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] bigTableValueColumns: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) className: VectorMapJoinOuterMultiKeyOperator native: true nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - outerSmallTableKeyMapping: 2 -> 42, 3 -> 43 - projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) - smallTableValueMapping: 35:int, 36:bigint, 37:string, 38:bigint, 39:string, 40:int, 41:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + outerSmallTableKeyMapping: 2 -> 40, 3 -> 41 + projectedOutput: 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2), 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) + smallTableValueMapping: 34:int, 35:bigint, 36:string, 37:bigint, 38:int, 39:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) hashTableImplementationType: OPTIMIZED - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50 input vertices: 1 Map 5 Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col40 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col33 (type: int), _col39 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col38 (type: int), _col28 (type: string), _col6 (type: int), _col50 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col34 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col45 (type: decimal(7,2)), _col47 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col35 (type: int), _col31 (type: int), _col41 (type: decimal(7,2)), _col36 (type: int), _col4 (type: int), _col48 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col29 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col30 (type: int), _col42 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col51 (type: decimal(7,2)), _col32 (type: int), _col37 (type: int), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49 + expressions: _col39 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col32 (type: int), _col38 (type: int), _col24 (type: int), _col21 (type: decimal(7,2)), _col26 (type: string), _col7 (type: int), _col3 (type: int), _col10 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col37 (type: int), _col6 (type: int), _col49 (type: decimal(7,2)), _col11 (type: decimal(7,2)), _col33 (type: int), _col17 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col44 (type: decimal(7,2)), _col46 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col8 (type: int), _col34 (type: int), _col30 (type: int), _col40 (type: decimal(7,2)), _col35 (type: int), _col4 (type: int), _col47 (type: decimal(7,2)), _col5 (type: int), _col13 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col43 (type: decimal(7,2)), _col42 (type: decimal(7,2)), _col1 (type: int), _col16 (type: decimal(7,2)), _col28 (type: int), _col2 (type: int), _col15 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col25 (type: bigint), _col9 (type: int), _col29 (type: int), _col41 (type: decimal(7,2)), _col0 (type: int), _col27 (type: bigint), _col50 (type: decimal(7,2)), _col31 (type: int), _col36 (type: int), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [51, 20, 44, 50, 35, 22, 37, 8, 4, 11, 60, 49, 39, 7, 61, 12, 45, 18, 15, 56, 58, 21, 9, 46, 42, 52, 47, 5, 59, 6, 14, 13, 55, 54, 2, 17, 40, 3, 16, 19, 36, 10, 41, 53, 1, 38, 62, 43, 48, 57] + projectedOutputColumnNums: [49, 20, 42, 48, 34, 22, 36, 8, 4, 11, 58, 47, 7, 59, 12, 43, 18, 15, 54, 56, 21, 9, 44, 40, 50, 45, 5, 57, 6, 14, 13, 53, 52, 2, 17, 38, 3, 16, 19, 35, 10, 39, 51, 1, 37, 60, 41, 46, 55] Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint), _col12 (type: string), 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), null (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint), 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), null (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [35, 36, 37, 38, 39, 31, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 63, 55, 56, 57, 58, 59, 60, 61, 62] - selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val null) -> 63:decimal(7,2) + projectedOutputColumnNums: [34, 35, 36, 37, 30, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 61, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 30:int, ConstantVectorExpression(val null) -> 61:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -412,17 +412,17 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 54:decimal(7,2))) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0), SelectColumnIsNull(col 52:decimal(7,2))) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col42 (type: int), _col24 (type: int), _col47 (type: int), _col2 (type: int), _col16 (type: int), _col23 (type: int), _col26 (type: int), _col48 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col43 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col32 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col49 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col28 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col46 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col41 (type: int), _col23 (type: int), _col46 (type: int), _col2 (type: int), _col15 (type: int), _col22 (type: int), _col25 (type: int), _col47 (type: int), _col11 (type: int), _col3 (type: int), _col0 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col42 (type: decimal(7,2)), 0 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col48 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col27 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col45 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [32, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 64, 55, 56, 57, 58, 59, 60, 61, 62] - selectExpressions: ConstantVectorExpression(val 2451181) -> 32:int, ConstantVectorExpression(val 0) -> 64:decimal(7,2) + projectedOutputColumnNums: [31, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 62, 53, 54, 55, 56, 57, 58, 59, 60] + selectExpressions: ConstantVectorExpression(val 2451181) -> 31:int, ConstantVectorExpression(val 0) -> 62:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -431,28 +431,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 43:int, 66:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 66:int + keyColumns: 41:int, 64:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 64:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 32:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:int, 50:int, 51:decimal(7,2), 52:decimal(7,2), 53:decimal(7,2), 64:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2), 61:decimal(7,2), 62:decimal(7,2) + valueColumns: 31:int, 39:int, 40:int, 41:int, 42:int, 43:int, 44:int, 45:int, 46:int, 47:int, 48:int, 49:decimal(7,2), 50:decimal(7,2), 51:decimal(7,2), 62:decimal(7,2), 53:decimal(7,2), 54:decimal(7,2), 55:decimal(7,2), 56:decimal(7,2), 57:decimal(7,2), 58:decimal(7,2), 59:decimal(7,2), 60:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 42:int), SelectColumnIsNull(col 43:int), SelectColumnIsNull(col 40:int)) - predicate: (_col24 is null and _col47 is null and _col36 is null) (type: boolean) + predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 40:int), SelectColumnIsNull(col 41:int), SelectColumnIsNull(col 38:int)) + predicate: (_col23 is null and _col46 is null and _col35 is null) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 2451181 (type: int), _col44 (type: int), _col34 (type: int), _col37 (type: int), _col8 (type: int), _col27 (type: int), _col29 (type: int), _col13 (type: int), _col7 (type: int), _col22 (type: int), _col41 (type: int), _col9 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col31 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col35 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col39 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col5 (type: decimal(7,2)) + expressions: 2451181 (type: int), _col43 (type: int), _col33 (type: int), _col36 (type: int), _col8 (type: int), _col26 (type: int), _col28 (type: int), _col12 (type: int), _col7 (type: int), _col21 (type: int), _col40 (type: int), _col9 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col30 (type: decimal(7,2)), _col29 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col37 (type: decimal(7,2)), _col34 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col38 (type: decimal(7,2)), _col1 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col5 (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [65, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 65:int + projectedOutputColumnNums: [63, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 63:int Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) @@ -461,27 +461,27 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), iceberg_bucket(_col2, 3) (type: int) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: 3:int, 67:int - keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 67:int + keyColumns: 3:int, 65:int + keyExpressions: VectorUDFAdaptor(iceberg_bucket(_col2, 3)) -> 65:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 65:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 63:int, 1:int, 2:int, 3:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2)), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)) Filter Operator Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 42:int, col 2:int), FilterLongColEqualLongColumn(col 43:int, col 3:int), FilterLongColEqualLongScalar(col 40:int, val 2451181), FilterLongColumnBetween(col 32:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 31:bigint, val 1000)(children: FuncFloorDoubleToLong(col 29:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 29:double) -> 31:bigint) -> 32:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) - predicate: ((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) (type: boolean) + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongColumn(col 40:int, col 2:int), FilterLongColEqualLongColumn(col 41:int, col 3:int), FilterLongColEqualLongScalar(col 38:int, val 2451181), FilterLongColumnBetween(col 31:bigint, left 1000, right 2000)(children: LongColMultiplyLongScalar(col 30:bigint, val 1000)(children: FuncFloorDoubleToLong(col 28:double)(children: LongColDivideLongScalar(col 2:int, val 1000) -> 28:double) -> 30:bigint) -> 31:bigint), FilterDecimalColLessDecimalScalar(col 14:decimal(7,2), val 0)) + predicate: ((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) (type: boolean) Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col6 (type: string), _col40 (type: bigint), _col45 (type: bigint) - outputColumnNames: _col4, _col6, _col40, _col45 + expressions: _col4 (type: int), _col6 (type: string), _col39 (type: bigint), _col44 (type: bigint) + outputColumnNames: _col4, _col6, _col39, _col44 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [35, 37, 36, 38] + projectedOutputColumnNums: [34, 36, 35, 37] Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -489,11 +489,11 @@ STAGE PLANS: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 35:int, col 36:bigint, col 37:string, col 38:bigint + keyExpressions: col 34:int, col 35:bigint, col 36:string, col 37:bigint native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] - keys: _col4 (type: int), _col40 (type: bigint), _col6 (type: string), _col45 (type: bigint) + keys: _col4 (type: int), _col39 (type: bigint), _col6 (type: string), _col44 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -526,7 +526,7 @@ STAGE PLANS: includeColumns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] dataColumns: ss_sold_date_sk:int, ss_sold_time_sk:int, ss_item_sk2:int, ss_customer_sk2:int, ss_cdemo_sk:int, ss_hdemo_sk:int, ss_addr_sk:int, ss_store_sk:int, ss_promo_sk:int, ss_ticket_number:int, ss_quantity:int, ss_wholesale_cost:decimal(7,2), ss_list_price:decimal(7,2), ss_sales_price:decimal(7,2), ss_ext_discount_amt:decimal(7,2), ss_ext_sales_price:decimal(7,2), ss_ext_wholesale_cost:decimal(7,2), ss_ext_list_price:decimal(7,2), ss_ext_tax:decimal(7,2), ss_coupon_amt:decimal(7,2), ss_net_paid:decimal(7,2), ss_net_paid_inc_tax:decimal(7,2), ss_net_profit:decimal(7,2) partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), decimal(7,2), bigint, bigint, bigint] Map 5 Map Operator Tree: TableScan @@ -535,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__PROJECTION:string, 28:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:ss_sold_date_sk:int, 1:ss_sold_time_sk:int, 2:ss_item_sk:int, 3:ss_customer_sk:int, 4:ss_cdemo_sk:int, 5:ss_hdemo_sk:int, 6:ss_addr_sk:int, 7:ss_store_sk:int, 8:ss_promo_sk:int, 9:ss_ticket_number:int, 10:ss_quantity:int, 11:ss_wholesale_cost:decimal(7,2), 12:ss_list_price:decimal(7,2), 13:ss_sales_price:decimal(7,2), 14:ss_ext_discount_amt:decimal(7,2), 15:ss_ext_sales_price:decimal(7,2), 16:ss_ext_wholesale_cost:decimal(7,2), 17:ss_ext_list_price:decimal(7,2), 18:ss_ext_tax:decimal(7,2), 19:ss_coupon_amt:decimal(7,2), 20:ss_net_paid:decimal(7,2), 21:ss_net_paid_inc_tax:decimal(7,2), 22:ss_net_profit:decimal(7,2), 23:PARTITION__SPEC__ID:int, 24:PARTITION__HASH:bigint, 25:FILE__PATH:string, 26:ROW__POSITION:bigint, 27:PARTITION__NAME:string] Filter Operator Filter Vectorization: className: VectorFilterOperator @@ -544,28 +544,28 @@ STAGE PLANS: predicate: ((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) (type: boolean) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 2451181 (type: int), ss_sold_time_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_cdemo_sk (type: int), ss_hdemo_sk (type: int), ss_addr_sk (type: int), ss_store_sk (type: int), ss_promo_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_list_price (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)), ss_ext_tax (type: decimal(7,2)), ss_coupon_amt (type: decimal(7,2)), ss_net_paid (type: decimal(7,2)), ss_net_paid_inc_tax (type: decimal(7,2)), ss_net_profit (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [23, 24, 25, 26, 27, 29, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - selectExpressions: ConstantVectorExpression(val 2451181) -> 29:int + projectedOutputColumnNums: [23, 24, 25, 26, 28, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + selectExpressions: ConstantVectorExpression(val 2451181) -> 28:int Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col8 (type: int), _col7 (type: int) + key expressions: _col6 (type: int), _col7 (type: int) null sort order: zz sort order: ++ - Map-reduce partition columns: _col7 (type: int) + Map-reduce partition columns: _col6 (type: int) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: 3:int, 2:int + keyColumns: 2:int, 3:int native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true partitionColumns: 2:int - valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 29:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) + valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2) Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)), _col27 (type: decimal(7,2)) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: decimal(7,2)), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decimal(7,2)), _col22 (type: decimal(7,2)), _col23 (type: decimal(7,2)), _col24 (type: decimal(7,2)), _col25 (type: decimal(7,2)), _col26 (type: decimal(7,2)) Execution mode: vectorized Map Vectorization: enabled: true @@ -909,16 +909,16 @@ Stage-6 Select Operator [SEL_50] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_46] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0) and _col32 is null) Select Operator [SEL_44] (rows=5 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48"] Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#) - BucketMapJoin:true,Conds:SEL_42._col2, _col1=RS_41._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51"] + BucketMapJoin:true,Conds:SEL_42._col1, _col2=RS_41._col6, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50"] <-Map 5 [CUSTOM_EDGE] vectorized MULTICAST [RS_41] - PartitionCols:_col7 + PartitionCols:_col6 Select Operator [SEL_40] (rows=2 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26"] Filter Operator [FIL_39] (rows=2 width=#Masked#) predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null) TableScan [TS_2] (rows=2 width=#Masked#) @@ -938,7 +938,7 @@ Stage-6 Select Operator [SEL_51] (rows=1 width=#Masked#) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] Filter Operator [FIL_47] (rows=1 width=#Masked#) - predicate:(_col24 is null and _col47 is null and _col36 is null) + predicate:(_col23 is null and _col46 is null and _col35 is null) Please refer to the previous Select Operator [SEL_44] Reducer 4 vectorized File Output Operator [FS_65] @@ -953,11 +953,11 @@ Stage-6 SHUFFLE [RS_57] PartitionCols:_col0, _col1, _col2, _col3 Group By Operator [GBY_56] (rows=1 width=#Masked#) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col40, _col6, _col45 + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col39, _col6, _col44 Select Operator [SEL_52] (rows=1 width=#Masked#) - Output:["_col4","_col6","_col40","_col45"] + Output:["_col4","_col6","_col39","_col44"] Filter Operator [FIL_48] (rows=1 width=#Masked#) - predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0)) + predicate:((_col23 = _col33) and (_col46 = _col36) and (_col35 = 2451181) and (floor((_col33 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col29 < 0)) Please refer to the previous Select Operator [SEL_44] Stage-7 Stats Work{} diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index e4275452b154..f94d7275f651 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -197,7 +197,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -351,7 +351,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -360,7 +360,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -467,7 +467,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -476,7 +476,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -797,7 +797,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 3e4c4a1e6e0a..734ac8f8f824 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -102,7 +102,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -282,7 +282,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -291,7 +291,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -442,7 +442,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -576,7 +576,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -688,7 +688,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -861,7 +861,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__PROJECTION:string, 17:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array, 2:arrayofarrays:array>, 3:arrayofmaps:array>, 4:arrayofstructs:array>, 5:mapofprimitives:map, 6:mapofarrays:map>, 7:mapofmaps:map>, 8:mapofstructs:map>, 9:structofprimitives:struct, 10:structofarrays:struct,birthdays:array>, 11:structofmaps:struct,map2:map>, 12:PARTITION__SPEC__ID:int, 13:PARTITION__HASH:bigint, 14:FILE__PATH:string, 15:ROW__POSITION:bigint, 16:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index 71cff814d471..c2a1c1c06d12 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__PROJECTION:string, 7:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:a:int, 1:b:string, 2:PARTITION__SPEC__ID:int, 3:PARTITION__HASH:bigint, 4:FILE__PATH:string, 5:ROW__POSITION:bigint, 6:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__PROJECTION:string, 15:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64, 10:PARTITION__SPEC__ID:int, 11:PARTITION__HASH:bigint, 12:FILE__PATH:string, 13:ROW__POSITION:bigint, 14:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true @@ -280,7 +280,7 @@ STAGE PLANS: aggregators: VectorUDAFMaxDouble(col 0:float) -> float className: VectorGroupByOperator groupByMode: HASH - keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 16:decimal(4,2) + keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 15:decimal(4,2) native: false vectorProcessingMode: HASH projectedOutputColumnNums: [0] @@ -480,7 +480,7 @@ STAGE PLANS: Map Operator Tree: TableScan Vectorization: native: true - vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__PROJECTION:string, 9:PARTITION__NAME:string] + vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string, 4:PARTITION__SPEC__ID:int, 5:PARTITION__HASH:bigint, 6:FILE__PATH:string, 7:ROW__POSITION:bigint, 8:PARTITION__NAME:string] Select Vectorization: className: VectorSelectOperator native: true diff --git a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out index 37f0589a9167..02111ac58daf 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/write_iceberg_branch.q.out @@ -239,8 +239,8 @@ STAGE PLANS: predicate: (a = 22) (type: boolean) Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), 22 (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), 22 (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 5 Data size: 960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -321,12 +321,12 @@ STAGE PLANS: predicate: (c = 66) (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), b (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col9 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), b (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col8 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), 66 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: int), _col5 (type: string), 66 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -337,7 +337,7 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Select Operator - expressions: 33 (type: int), _col9 (type: string), 66 (type: int) + expressions: 33 (type: int), _col8 (type: string), 66 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -456,16 +456,16 @@ STAGE PLANS: predicate: a is not null (type: boolean) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), PARTITION__PROJECTION (type: string), a (type: int), b (type: string), c (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), a (type: int), b (type: string), c (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col5 (type: int) + key expressions: _col4 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col5 (type: int) + Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 4 Data size: 768 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col6 (type: string), _col7 (type: int) + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col5 (type: string), _col6 (type: int) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -474,19 +474,19 @@ STAGE PLANS: Left Outer Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col5 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + 1 _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col7 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + expressions: _col1 (type: string), _col0 (type: int), _col5 (type: string), _col2 (type: int), _col6 (type: bigint), _col4 (type: bigint), _col3 (type: int), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 6 Data size: 633 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col10 = _col1) and (_col10 > 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 > 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -497,11 +497,11 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint), _col3 (type: string), _col10 (type: int), _col9 (type: string), _col8 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint), _col9 (type: int), _col8 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -512,10 +512,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: ((_col10 = _col1) and (_col10 <= 100)) (type: boolean) + predicate: ((_col9 = _col1) and (_col9 <= 100)) (type: boolean) Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col10 (type: int), 'Merged' (type: string), (_col8 + 10) (type: int) + expressions: _col9 (type: int), 'Merged' (type: string), (_col7 + 10) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -527,10 +527,10 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: _col10 is null (type: boolean) + predicate: _col9 is null (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: int), _col0 (type: string), _col4 (type: int) + expressions: _col1 (type: int), _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -542,15 +542,15 @@ STAGE PLANS: serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe name: default.ice01 Filter Operator - predicate: (_col10 = _col1) (type: boolean) + predicate: (_col9 = _col1) (type: boolean) Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: int) - outputColumnNames: _col2, _col5, _col6, _col7 + expressions: _col2 (type: string), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: int) + outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 3 Data size: 316 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col7 (type: int), _col6 (type: bigint), _col2 (type: string), _col5 (type: bigint) + keys: _col6 (type: int), _col5 (type: bigint), _col2 (type: string), _col4 (type: bigint) minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 64bd708c37ea..b07470fb5374 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -680,12 +680,6 @@ public static Object[] populateVirtualColumnValues(ExecMapperContext ctx, vcValues[i] = new LongWritable(ctx.getIoCxt().getPositionDeleteInfo().getPartitionHash()); } break; - case PARTITION_PROJECTION: - vcValues[i] = null; - if (ctx.getIoCxt().getPositionDeleteInfo() != null) { - vcValues[i] = new Text(ctx.getIoCxt().getPositionDeleteInfo().getPartitionProjection()); - } - break; case PARTITION_NAME: vcValues[i] = null; if (ctx.getIoCxt().getPartitionName() != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java index f5e531496734..d3485a49f713 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java @@ -27,38 +27,32 @@ public class PositionDeleteInfo { private static final String CONF_KEY_PART_HASH = "hive.io.context.position.delete.partition.hash"; private static final String CONF_KEY_FILE_PATH = "hive.io.context.position.delete.file.path"; private static final String CONF_KEY_ROW_POSITION = "hive.io.context.position.delete.row.position"; - private static final String CONF_KEY_PARTITION_PROJECTION = "hive.io.context.position.delete.partition.projection"; public static PositionDeleteInfo parseFromConf(Configuration conf) { int specId = conf.getInt(CONF_KEY_SPEC_ID, -1); long partHash = conf.getLong(CONF_KEY_PART_HASH, -1); String filePath = conf.get(CONF_KEY_FILE_PATH); long rowPos = conf.getLong(CONF_KEY_ROW_POSITION, -1); - String partitionProjection = conf.get(CONF_KEY_PARTITION_PROJECTION); - return new PositionDeleteInfo(specId, partHash, filePath, rowPos, partitionProjection); + return new PositionDeleteInfo(specId, partHash, filePath, rowPos); } - public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos, - String partitionProjection) { + public static void setIntoConf(Configuration conf, int specId, long partHash, String filePath, long filePos) { conf.setInt(CONF_KEY_SPEC_ID, specId); conf.setLong(CONF_KEY_PART_HASH, partHash); conf.set(CONF_KEY_FILE_PATH, filePath); conf.setLong(CONF_KEY_ROW_POSITION, filePos); - conf.set(CONF_KEY_PARTITION_PROJECTION, partitionProjection); } private final int specId; private final long partitionHash; private final String filePath; private final long filePos; - private final String partitionProjection; - public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos, String partitionProjection) { + public PositionDeleteInfo(int specId, long partitionHash, String filePath, long filePos) { this.specId = specId; this.partitionHash = partitionHash; this.filePath = filePath; this.filePos = filePos; - this.partitionProjection = partitionProjection; } public int getSpecId() { @@ -76,8 +70,4 @@ public String getFilePath() { public long getFilePos() { return filePos; } - - public String getPartitionProjection() { - return partitionProjection; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index fb5852cf144c..4fd718af61be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -57,7 +57,6 @@ public enum VirtualColumn { FILE_PATH("FILE__PATH", TypeInfoFactory.stringTypeInfo), ROW_POSITION("ROW__POSITION", TypeInfoFactory.longTypeInfo), SNAPSHOT_ID("SNAPSHOT__ID", TypeInfoFactory.longTypeInfo), - PARTITION_PROJECTION("PARTITION__PROJECTION", TypeInfoFactory.stringTypeInfo), PARTITION_NAME("PARTITION__NAME", TypeInfoFactory.stringTypeInfo), ROW_LINEAGE_ID("ROW__LINEAGE__ID", TypeInfoFactory.longTypeInfo), LAST_UPDATED_SEQUENCE_NUMBER("LAST__UPDATED__SEQUENCE__NUMBER", TypeInfoFactory.longTypeInfo), @@ -75,7 +74,7 @@ public enum VirtualColumn { ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName(), ROWISDELETED.getName(), PARTITION_SPEC_ID.getName(), PARTITION_HASH.getName(), FILE_PATH.getName(), ROW_POSITION.getName(), - PARTITION_PROJECTION.getName(), PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), + PARTITION_NAME.getName(), ROW_LINEAGE_ID.getName(), LAST_UPDATED_SEQUENCE_NUMBER.getName()); public static final ImmutableMap VIRTUAL_COLUMN_NAME_MAP = diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 7d0a3cb2a8cc..12c43f7f6e8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -309,7 +309,6 @@ public class Vectorizer implements PhysicalPlanResolver { VirtualColumn.PARTITION_HASH, VirtualColumn.FILE_PATH, VirtualColumn.ROW_POSITION, - VirtualColumn.PARTITION_PROJECTION, VirtualColumn.PARTITION_NAME, VirtualColumn.ROW_LINEAGE_ID, VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER);