From 915465dc4473d4944dbe59ea4304560dfe88dbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 7 Apr 2026 10:53:18 +0800 Subject: [PATCH 1/3] [core] Introduce BLOB_REF for shared blob data --- .../java/org/apache/paimon/CoreOptions.java | 25 ++- .../org/apache/paimon/types/BlobRefType.java | 65 +++++++ .../paimon/types/DataTypeDefaultVisitor.java | 5 + .../paimon/types/DataTypeJsonParser.java | 3 + .../org/apache/paimon/types/DataTypeRoot.java | 2 + .../apache/paimon/types/DataTypeVisitor.java | 2 + .../org/apache/paimon/types/DataTypes.java | 4 + .../arrow/ArrowFieldTypeConversion.java | 6 + .../Arrow2PaimonVectorConverter.java | 6 + .../ArrowFieldWriterFactoryVisitor.java | 6 + .../org/apache/paimon/data/BinaryWriter.java | 9 + .../java/org/apache/paimon/data/Blob.java | 8 + .../org/apache/paimon/data/BlobReference.java | 141 ++++++++++++++ .../apache/paimon/data/BlobReferenceBlob.java | 98 ++++++++++ .../paimon/data/BlobReferenceInternalRow.java | 160 ++++++++++++++++ .../paimon/data/BlobReferenceResolver.java | 28 +++ .../org/apache/paimon/data/BlobUtils.java | 67 +++++++ .../org/apache/paimon/data/InternalRow.java | 4 + .../data/columnar/RowToColumnConverter.java | 6 + .../data/serializer/BlobRefSerializer.java | 52 +++++ .../data/serializer/InternalSerializers.java | 2 + .../fileindex/bitmap/BitmapTypeVisitor.java | 6 + .../fileindex/bloomfilter/FastHash.java | 6 + .../paimon/sort/hilbert/HilbertIndexer.java | 6 + .../apache/paimon/sort/zorder/ZIndexer.java | 6 + .../types/InternalRowToSizeVisitor.java | 17 +- .../paimon/utils/VectorMappingUtils.java | 6 + .../paimon/data/BlobReferenceBlobTest.java | 53 ++++++ .../apache/paimon/data/BlobReferenceTest.java | 53 ++++++ .../types/InternalRowToSizeVisitorTest.java | 19 ++ .../paimon/schema/SchemaValidation.java | 55 +++++- .../table/AppendOnlyFileStoreTable.java | 3 +- .../table/PrimaryKeyFileStoreTable.java | 5 +- .../table/source/AbstractDataTableRead.java | 30 ++- .../paimon/table/source/AppendTableRead.java | 6 +- .../table/source/KeyValueTableRead.java | 6 +- .../paimon/utils/BlobReferenceLookup.java | 91 +++++++++ .../schema/BlobRefSchemaValidationTest.java | 69 +++++++ .../paimon/utils/BlobReferenceLookupTest.java | 179 ++++++++++++++++++ .../paimon/flink/DataTypeToLogicalType.java | 6 + .../org/apache/paimon/flink/FlinkCatalog.java | 15 ++ .../apache/paimon/flink/FlinkRowWrapper.java | 22 +-- .../paimon/flink/LogicalTypeConversion.java | 8 + .../flink/lookup/LookupCompactDiffRead.java | 6 +- .../flink/lookup/LookupFileStoreTable.java | 4 +- .../source/FileStoreSourceSplitReader.java | 3 +- .../source/TestChangelogDataReadWrite.java | 2 +- .../format/avro/AvroSchemaConverter.java | 1 + .../format/avro/FieldReaderFactory.java | 11 +- .../format/avro/FieldWriterFactory.java | 21 +- .../apache/paimon/format/orc/OrcTypeUtil.java | 1 + .../format/orc/writer/FieldWriterFactory.java | 13 ++ .../parquet/ParquetSchemaConverter.java | 1 + .../parquet/reader/ParquetReaderUtil.java | 4 +- .../reader/ParquetVectorUpdaterFactory.java | 6 + .../parquet/writer/ParquetRowDataWriter.java | 22 +++ .../filter2/predicate/ParquetFilters.java | 6 + .../format/avro/AvroFileFormatTest.java | 34 ++++ .../paimon/format/orc/OrcTypeUtilTest.java | 1 + .../parquet/ParquetSchemaConverterTest.java | 13 ++ .../org/apache/paimon/hive/HiveTypeUtils.java | 6 + .../paimon/format/lance/LanceFileFormat.java | 6 + .../org/apache/paimon/spark/SparkCatalog.java | 7 + .../paimon/spark/SparkInternalRowWrapper.java | 19 +- .../org/apache/paimon/spark/SparkRow.java | 21 +- .../apache/paimon/spark/SparkTypeUtils.java | 6 + .../paimon/spark/data/SparkInternalRow.scala | 3 +- .../format/vortex/VortexFileFormat.java | 7 + 68 files changed, 1527 insertions(+), 63 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/types/BlobRefType.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceResolver.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/BlobRefSchemaValidationTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 897bebcf686c..8b38b2d8ee0f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2254,6 +2254,14 @@ public InlineElement getDescription() { "Specifies column names that should be stored as blob type. " + "This is used when you want to treat a BYTES column as a BLOB."); + public static final ConfigOption BLOB_REF_FIELD = + key("blob-ref-field") + .stringType() + .noDefaultValue() + .withDescription( + "Specifies column names that should be stored as blob reference type. " + + "This is used when you want to treat a BYTES column as a BLOB_REF."); + @Immutable public static final ConfigOption BLOB_DESCRIPTOR_FIELD = key("blob-descriptor-field") @@ -2935,7 +2943,13 @@ public Set blobExternalStorageField() { * subset of descriptor fields and therefore are also updatable. */ public Set updatableBlobFields() { - return blobDescriptorField(); + Set fields = new HashSet<>(blobDescriptorField()); + fields.addAll(blobRefField()); + return fields; + } + + public Set blobRefField() { + return parseCommaSeparatedSet(BLOB_REF_FIELD); } /** @@ -3274,6 +3288,15 @@ public static List blobField(Map options) { return Arrays.stream(string.split(",")).map(String::trim).collect(Collectors.toList()); } + public static List blobRefField(Map options) { + String string = options.get(BLOB_REF_FIELD.key()); + if (string == null) { + return Collections.emptyList(); + } + + return Arrays.stream(string.split(",")).map(String::trim).collect(Collectors.toList()); + } + public boolean sequenceFieldSortOrderIsAscending() { return options.get(SEQUENCE_FIELD_SORT_ORDER) == SortOrder.ASCENDING; } diff --git a/paimon-api/src/main/java/org/apache/paimon/types/BlobRefType.java b/paimon-api/src/main/java/org/apache/paimon/types/BlobRefType.java new file mode 100644 index 000000000000..e5d53ead4e4a --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/types/BlobRefType.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import org.apache.paimon.annotation.Public; + +/** + * Data type of blob reference. + * + *

{@link BlobRefType} stores reference bytes inline in data files instead of writing payloads to + * Paimon-managed {@code .blob} files. + * + * @since 1.5.0 + */ +@Public +public final class BlobRefType extends DataType { + + private static final long serialVersionUID = 1L; + + private static final String FORMAT = "BLOB_REF"; + + public BlobRefType(boolean isNullable) { + super(isNullable, DataTypeRoot.BLOB_REF); + } + + public BlobRefType() { + this(true); + } + + @Override + public int defaultSize() { + return BlobType.DEFAULT_SIZE; + } + + @Override + public DataType copy(boolean isNullable) { + return new BlobRefType(isNullable); + } + + @Override + public String asSQLString() { + return withNullability(FORMAT); + } + + @Override + public R accept(DataTypeVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java index af680ede62e2..4a819d42ae2c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java @@ -119,6 +119,11 @@ public R visit(BlobType blobType) { return defaultMethod(blobType); } + @Override + public R visit(BlobRefType blobRefType) { + return defaultMethod(blobRefType); + } + @Override public R visit(ArrayType arrayType) { return defaultMethod(arrayType); diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java index 4079dd8c47c0..5e2a39a29fcd 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java @@ -331,6 +331,7 @@ private enum Keyword { LEGACY, VARIANT, BLOB, + BLOB_REF, NOT } @@ -549,6 +550,8 @@ private DataType parseTypeByKeyword() { return new VariantType(); case BLOB: return new BlobType(); + case BLOB_REF: + return new BlobRefType(); case VECTOR: return parseVectorType(); default: diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java index f55da9c4706f..27f8d65a40bf 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java @@ -104,6 +104,8 @@ public enum DataTypeRoot { BLOB(DataTypeFamily.PREDEFINED), + BLOB_REF(DataTypeFamily.PREDEFINED), + ARRAY(DataTypeFamily.CONSTRUCTED, DataTypeFamily.COLLECTION), VECTOR(DataTypeFamily.CONSTRUCTED, DataTypeFamily.COLLECTION), diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java index 6e377309f237..074a1d82ec70 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java @@ -66,6 +66,8 @@ public interface DataTypeVisitor { R visit(BlobType blobType); + R visit(BlobRefType blobRefType); + R visit(ArrayType arrayType); R visit(VectorType vectorType); diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java index 39b180651ef5..0033984bc6cc 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java @@ -163,6 +163,10 @@ public static BlobType BLOB() { return new BlobType(); } + public static BlobRefType BLOB_REF() { + return new BlobRefType(); + } + public static OptionalInt getPrecision(DataType dataType) { return dataType.accept(PRECISION_EXTRACTOR); } diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java index 33defc8f9a01..37b36a24d154 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java @@ -21,6 +21,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -163,6 +164,11 @@ public FieldType visit(BlobType blobType) { throw new UnsupportedOperationException(); } + @Override + public FieldType visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException(); + } + private TimeUnit getTimeUnit(int precision) { if (precision == 0) { return TimeUnit.SECOND; diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java index e1fe66883a84..d8672dfdc23b 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java @@ -47,6 +47,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -447,6 +448,11 @@ public Arrow2PaimonVectorConverter visit(BlobType blobType) { throw new UnsupportedOperationException(); } + @Override + public Arrow2PaimonVectorConverter visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException(); + } + @Override public Arrow2PaimonVectorConverter visit(ArrayType arrayType) { final Arrow2PaimonVectorConverter arrowVectorConvertor = diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java index ccff6d6a24f6..b4d38e2dae61 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java @@ -21,6 +21,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -156,6 +157,11 @@ public ArrowFieldWriterFactory visit(BlobType blobType) { throw new UnsupportedOperationException("Doesn't support BlobType."); } + @Override + public ArrowFieldWriterFactory visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException("Doesn't support BlobRefType."); + } + @Override public ArrowFieldWriterFactory visit(ArrayType arrayType) { ArrowFieldWriterFactory elementWriterFactory = arrayType.getElementType().accept(this); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java index 2e0cd5701b71..b22336e31b8d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java @@ -157,6 +157,10 @@ static void write( case BLOB: writer.writeBlob(pos, (Blob) o); break; + case BLOB_REF: + byte[] refBytes = BlobUtils.serializeBlobReference((Blob) o); + writer.writeBinary(pos, refBytes, 0, refBytes.length); + break; default: throw new UnsupportedOperationException("Not support type: " + type); } @@ -241,6 +245,11 @@ static ValueSetter createValueSetter(DataType elementType, Serializer seriali return (writer, pos, value) -> writer.writeVariant(pos, (Variant) value); case BLOB: return (writer, pos, value) -> writer.writeBlob(pos, (Blob) value); + case BLOB_REF: + return (writer, pos, value) -> { + byte[] bytes = BlobUtils.serializeBlobReference((Blob) value); + writer.writeBinary(pos, bytes, 0, bytes.length); + }; default: String msg = String.format( diff --git a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java index 6586124e466b..88733d884d45 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.utils.UriReader; +import org.apache.paimon.utils.UriReaderFactory; import java.io.IOException; import java.util.function.Supplier; @@ -65,6 +66,13 @@ static Blob fromDescriptor(UriReader reader, BlobDescriptor descriptor) { return new BlobRef(reader, descriptor); } + static Blob fromReference( + UriReaderFactory uriReaderFactory, + BlobReferenceResolver fallbackResolver, + BlobReference reference) { + return new BlobReferenceBlob(uriReaderFactory, fallbackResolver, reference); + } + static Blob fromInputStream(Supplier supplier) { return new BlobStream(supplier); } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java new file mode 100644 index 000000000000..8e259e0ef895 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Objects; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** Serialized metadata for a {@code BLOB_REF} field. */ +public class BlobReference implements Serializable { + + private static final long serialVersionUID = 1L; + + private static final long MAGIC = 0x424C4F4252454631L; // "BLOBREF1" + private static final byte CURRENT_VERSION = 1; + + private final BlobDescriptor descriptor; + private final String tableName; + private final int fieldId; + private final long rowId; + + public BlobReference(BlobDescriptor descriptor, String tableName, int fieldId, long rowId) { + this.descriptor = descriptor; + this.tableName = tableName; + this.fieldId = fieldId; + this.rowId = rowId; + } + + public BlobDescriptor descriptor() { + return descriptor; + } + + public String tableName() { + return tableName; + } + + public int fieldId() { + return fieldId; + } + + public long rowId() { + return rowId; + } + + public byte[] serialize() { + byte[] descriptorBytes = descriptor.serialize(); + byte[] tableBytes = tableName.getBytes(UTF_8); + + int totalSize = 1 + 8 + 4 + descriptorBytes.length + 4 + tableBytes.length + 4 + 8; + ByteBuffer buffer = ByteBuffer.allocate(totalSize).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(CURRENT_VERSION); + buffer.putLong(MAGIC); + buffer.putInt(descriptorBytes.length); + buffer.put(descriptorBytes); + buffer.putInt(tableBytes.length); + buffer.put(tableBytes); + buffer.putInt(fieldId); + buffer.putLong(rowId); + return buffer.array(); + } + + public static BlobReference deserialize(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + byte version = buffer.get(); + if (version != CURRENT_VERSION) { + throw new UnsupportedOperationException( + "Expecting BlobReference version to be " + + CURRENT_VERSION + + ", but found " + + version + + "."); + } + + long magic = buffer.getLong(); + if (magic != MAGIC) { + throw new IllegalArgumentException( + "Invalid BlobReference: missing magic header. Expected magic: " + + MAGIC + + ", but found: " + + magic); + } + + byte[] descriptorBytes = new byte[buffer.getInt()]; + buffer.get(descriptorBytes); + + byte[] tableBytes = new byte[buffer.getInt()]; + buffer.get(tableBytes); + + int fieldId = buffer.getInt(); + long rowId = buffer.getLong(); + return new BlobReference( + BlobDescriptor.deserialize(descriptorBytes), + new String(tableBytes, UTF_8), + fieldId, + rowId); + } + + public static boolean isBlobReference(byte[] bytes) { + if (bytes == null || bytes.length < 9) { + return false; + } + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + return buffer.get() == CURRENT_VERSION && MAGIC == buffer.getLong(); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + BlobReference that = (BlobReference) o; + return fieldId == that.fieldId + && rowId == that.rowId + && Objects.equals(descriptor, that.descriptor) + && Objects.equals(tableName, that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(descriptor, tableName, fieldId, rowId); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java new file mode 100644 index 000000000000..b42cd10a0b4e --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.UriReaderFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Objects; + +/** {@link Blob} implementation backed by a serialized {@link BlobReference}. */ +public class BlobReferenceBlob implements Blob, Serializable { + + private static final long serialVersionUID = 1L; + + private final @Nullable BlobReferenceResolver fallbackResolver; + private final @Nullable UriReaderFactory uriReaderFactory; + private final BlobReference reference; + + public BlobReferenceBlob( + @Nullable UriReaderFactory uriReaderFactory, + @Nullable BlobReferenceResolver fallbackResolver, + BlobReference reference) { + this.uriReaderFactory = uriReaderFactory; + this.fallbackResolver = fallbackResolver; + this.reference = reference; + } + + public BlobReference reference() { + return reference; + } + + @Override + public byte[] toData() { + try { + return IOUtils.readFully(newInputStream(), true); + } catch (IOException e) { + throw new RuntimeException("Failed to read blob reference data.", e); + } + } + + @Override + public BlobDescriptor toDescriptor() { + return reference.descriptor(); + } + + @Override + public SeekableInputStream newInputStream() throws IOException { + if (uriReaderFactory != null) { + try { + return Blob.fromDescriptor( + uriReaderFactory.create(reference.descriptor().uri()), + reference.descriptor()) + .newInputStream(); + } catch (Exception ignored) { + // Fall through to the metadata-based lookup. + } + } + if (fallbackResolver == null) { + throw new IOException("Blob reference cannot be resolved without fallback resolver."); + } + return fallbackResolver.resolve(reference).newInputStream(); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + BlobReferenceBlob that = (BlobReferenceBlob) o; + return Objects.equals(reference, that.reference); + } + + @Override + public int hashCode() { + return Objects.hash(reference); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java new file mode 100644 index 000000000000..1f3e1a14c551 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.types.RowKind; +import org.apache.paimon.utils.UriReaderFactory; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; + +/** {@link InternalRow} wrapper which resolves {@code BLOB_REF} bytes lazily on {@link #getBlob}. */ +public class BlobReferenceInternalRow implements InternalRow, Serializable { + + private static final long serialVersionUID = 1L; + + private final InternalRow wrapped; + private final Set blobRefFields; + private final UriReaderFactory uriReaderFactory; + private final BlobReferenceResolver fallbackResolver; + + public BlobReferenceInternalRow( + InternalRow wrapped, + int[] blobRefFields, + UriReaderFactory uriReaderFactory, + BlobReferenceResolver fallbackResolver) { + this.wrapped = wrapped; + this.blobRefFields = new HashSet<>(); + for (int field : blobRefFields) { + this.blobRefFields.add(field); + } + this.uriReaderFactory = uriReaderFactory; + this.fallbackResolver = fallbackResolver; + } + + @Override + public int getFieldCount() { + return wrapped.getFieldCount(); + } + + @Override + public RowKind getRowKind() { + return wrapped.getRowKind(); + } + + @Override + public void setRowKind(RowKind kind) { + wrapped.setRowKind(kind); + } + + @Override + public boolean isNullAt(int pos) { + return wrapped.isNullAt(pos); + } + + @Override + public boolean getBoolean(int pos) { + return wrapped.getBoolean(pos); + } + + @Override + public byte getByte(int pos) { + return wrapped.getByte(pos); + } + + @Override + public short getShort(int pos) { + return wrapped.getShort(pos); + } + + @Override + public int getInt(int pos) { + return wrapped.getInt(pos); + } + + @Override + public long getLong(int pos) { + return wrapped.getLong(pos); + } + + @Override + public float getFloat(int pos) { + return wrapped.getFloat(pos); + } + + @Override + public double getDouble(int pos) { + return wrapped.getDouble(pos); + } + + @Override + public BinaryString getString(int pos) { + return wrapped.getString(pos); + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + return wrapped.getDecimal(pos, precision, scale); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + return wrapped.getTimestamp(pos, precision); + } + + @Override + public byte[] getBinary(int pos) { + return wrapped.getBinary(pos); + } + + @Override + public org.apache.paimon.data.variant.Variant getVariant(int pos) { + return wrapped.getVariant(pos); + } + + @Override + public Blob getBlob(int pos) { + if (!blobRefFields.contains(pos)) { + return wrapped.getBlob(pos); + } + return BlobUtils.fromBytes( + wrapped.getBinary(pos), uriReaderFactory, fallbackResolver, null); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + return wrapped.getRow(pos, numFields); + } + + @Override + public InternalArray getArray(int pos) { + return wrapped.getArray(pos); + } + + @Override + public InternalVector getVector(int pos) { + return wrapped.getVector(pos); + } + + @Override + public InternalMap getMap(int pos) { + return wrapped.getMap(pos); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceResolver.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceResolver.java new file mode 100644 index 000000000000..9e0d78485542 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceResolver.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import java.io.Serializable; + +/** Resolves a {@link BlobReference} through fallback metadata. */ +@FunctionalInterface +public interface BlobReferenceResolver extends Serializable { + + Blob resolve(BlobReference reference); +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java new file mode 100644 index 000000000000..8acf8ee669a7 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.utils.UriReader; +import org.apache.paimon.utils.UriReaderFactory; + +import javax.annotation.Nullable; + +/** Utilities for decoding and encoding blob-related bytes. */ +public class BlobUtils { + + public static Blob fromBytes( + byte[] bytes, + @Nullable UriReaderFactory uriReaderFactory, + @Nullable BlobReferenceResolver fallbackResolver, + @Nullable FileIO fileIO) { + if (BlobReference.isBlobReference(bytes)) { + if (uriReaderFactory == null || fallbackResolver == null) { + throw new IllegalStateException( + "Blob reference bytes require both uri reader factory and fallback resolver."); + } + return new BlobReferenceBlob( + uriReaderFactory, fallbackResolver, BlobReference.deserialize(bytes)); + } + + if (BlobDescriptor.isBlobDescriptor(bytes)) { + BlobDescriptor descriptor = BlobDescriptor.deserialize(bytes); + UriReader reader = + uriReaderFactory != null + ? uriReaderFactory.create(descriptor.uri()) + : UriReader.fromFile(fileIO); + return Blob.fromDescriptor(reader, descriptor); + } + + return new BlobData(bytes); + } + + public static byte[] serializeBlobReference(Blob blob) { + if (!(blob instanceof BlobReferenceBlob)) { + throw new IllegalArgumentException( + "BLOB_REF fields only accept BlobReferenceBlob values, but found " + + blob.getClass().getSimpleName() + + "."); + } + return ((BlobReferenceBlob) blob).reference().serialize(); + } + + private BlobUtils() {} +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java index 3bbb85f49963..f4e9e6960b6b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java @@ -146,6 +146,9 @@ static Class getDataClass(DataType type) { case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return Timestamp.class; + case BLOB: + case BLOB_REF: + return Blob.class; case ARRAY: return InternalArray.class; case MULTISET: @@ -228,6 +231,7 @@ static FieldGetter createFieldGetter(DataType fieldType, int fieldPos) { fieldGetter = row -> row.getVariant(fieldPos); break; case BLOB: + case BLOB_REF: fieldGetter = row -> row.getBlob(fieldPos); break; default: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java index de962ad86a39..12b7a567ec65 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java @@ -41,6 +41,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -262,6 +263,11 @@ public TypeConverter visit(BlobType blobType) { throw new UnsupportedOperationException(); } + @Override + public TypeConverter visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException(); + } + @Override public TypeConverter visit(ArrayType arrayType) { return createConverter( diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java new file mode 100644 index 000000000000..03c304ed1cce --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.serializer; + +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.BlobReferenceBlob; +import org.apache.paimon.data.BlobUtils; +import org.apache.paimon.io.DataInputView; +import org.apache.paimon.io.DataOutputView; + +import java.io.IOException; + +/** Type serializer for {@code BLOB_REF}. */ +public class BlobRefSerializer extends SerializerSingleton { + + private static final long serialVersionUID = 1L; + + public static final BlobRefSerializer INSTANCE = new BlobRefSerializer(); + + @Override + public Blob copy(Blob from) { + return from; + } + + @Override + public void serialize(Blob blob, DataOutputView target) throws IOException { + BinarySerializer.INSTANCE.serialize(BlobUtils.serializeBlobReference(blob), target); + } + + @Override + public Blob deserialize(DataInputView source) throws IOException { + byte[] bytes = BinarySerializer.INSTANCE.deserialize(source); + return new BlobReferenceBlob(null, null, BlobReference.deserialize(bytes)); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java index 6669f347ff27..9d4c9dba1798 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java @@ -92,6 +92,8 @@ private static Serializer createInternal(DataType type) { return VariantSerializer.INSTANCE; case BLOB: return BlobSerializer.INSTANCE; + case BLOB_REF: + return BlobRefSerializer.INSTANCE; default: throw new UnsupportedOperationException( "Unsupported type '" + type + "' to get internal serializer"); diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapTypeVisitor.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapTypeVisitor.java index 4183bfbb2bf8..57fcc8665b97 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapTypeVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapTypeVisitor.java @@ -21,6 +21,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -176,4 +177,9 @@ public final R visit(VariantType rowType) { public final R visit(BlobType blobType) { throw new UnsupportedOperationException("Does not support type blob"); } + + @Override + public final R visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException("Does not support type blob ref"); + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/FastHash.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/FastHash.java index 322847f849ab..722ab63bc4f0 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/FastHash.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/FastHash.java @@ -23,6 +23,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -172,6 +173,11 @@ public FastHash visit(BlobType blobType) { throw new UnsupportedOperationException("Does not support type blob"); } + @Override + public FastHash visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException("Does not support type blob_ref"); + } + @Override public FastHash visit(ArrayType arrayType) { throw new UnsupportedOperationException("Does not support type array"); diff --git a/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java b/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java index 241dc6100379..254204dc2511 100644 --- a/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/sort/hilbert/HilbertIndexer.java @@ -25,6 +25,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -272,6 +273,11 @@ public HProcessFunction visit(BlobType blobType) { throw new RuntimeException("Unsupported type"); } + @Override + public HProcessFunction visit(BlobRefType blobRefType) { + throw new RuntimeException("Unsupported type"); + } + @Override public HProcessFunction visit(ArrayType arrayType) { throw new RuntimeException("Unsupported type"); diff --git a/paimon-common/src/main/java/org/apache/paimon/sort/zorder/ZIndexer.java b/paimon-common/src/main/java/org/apache/paimon/sort/zorder/ZIndexer.java index 1d40fe75e776..f95e767cb5ae 100644 --- a/paimon-common/src/main/java/org/apache/paimon/sort/zorder/ZIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/sort/zorder/ZIndexer.java @@ -26,6 +26,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -360,6 +361,11 @@ public ZProcessFunction visit(BlobType blobType) { throw new UnsupportedOperationException("Does not support type blob"); } + @Override + public ZProcessFunction visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException("Does not support type blob_ref"); + } + @Override public ZProcessFunction visit(ArrayType arrayType) { throw new RuntimeException("Unsupported type"); diff --git a/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java b/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java index dbac55a07dde..94c71b6346ec 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java @@ -18,6 +18,7 @@ package org.apache.paimon.types; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.DataGetters; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; @@ -229,7 +230,21 @@ public BiFunction visit(BlobType blobType) { if (row.isNullAt(index)) { return NULL_SIZE; } else { - return Math.toIntExact(row.getVariant(index).sizeInBytes()); + return row.getBlob(index).toData().length; + } + }; + } + + @Override + public BiFunction visit(BlobRefType blobRefType) { + return (row, index) -> { + if (row.isNullAt(index)) { + return NULL_SIZE; + } + try { + return row.getBinary(index).length; + } catch (ClassCastException | UnsupportedOperationException e) { + return BlobUtils.serializeBlobReference(row.getBlob(index)).length; } }; } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java index 99e8fd455c41..6ea9a0a7b52f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java @@ -45,6 +45,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -336,6 +337,11 @@ public ColumnVector visit(BlobType blobType) { throw new UnsupportedOperationException("BlobType is not supported."); } + @Override + public ColumnVector visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException("BlobRefType is not supported."); + } + @Override public ColumnVector visit(ArrayType arrayType) { return new ArrayColumnVector() { diff --git a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java new file mode 100644 index 000000000000..c57d9509e791 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.apache.paimon.fs.SeekableInputStream; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Test for {@link BlobReferenceBlob}. */ +public class BlobReferenceBlobTest { + + @Test + public void testFallbackKeepsStreamingInput() throws Exception { + BlobReference reference = + new BlobReference( + new BlobDescriptor("file:/missing", 0L, -1L), "default.source", 7, 5L); + BlobReferenceResolver resolver = mock(BlobReferenceResolver.class); + Blob resolved = mock(Blob.class); + SeekableInputStream inputStream = mock(SeekableInputStream.class); + when(resolver.resolve(reference)).thenReturn(resolved); + when(resolved.newInputStream()).thenReturn(inputStream); + + SeekableInputStream actual = + new BlobReferenceBlob(null, resolver, reference).newInputStream(); + + assertThat(actual).isSameAs(inputStream); + verify(resolver).resolve(reference); + verify(resolved).newInputStream(); + verify(resolved, never()).toData(); + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java new file mode 100644 index 000000000000..0ba357b7e009 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Test for {@link BlobReference}. */ +public class BlobReferenceTest { + + @Test + public void testSerializeAndDeserialize() { + BlobDescriptor descriptor = new BlobDescriptor("/test/path", 10L, 20L); + BlobReference reference = new BlobReference(descriptor, "default.source", 7, 5L); + + BlobReference deserialized = BlobReference.deserialize(reference.serialize()); + + assertThat(deserialized.descriptor()).isEqualTo(descriptor); + assertThat(deserialized.tableName()).isEqualTo("default.source"); + assertThat(deserialized.fieldId()).isEqualTo(7); + assertThat(deserialized.rowId()).isEqualTo(5L); + } + + @Test + public void testRejectUnexpectedVersion() { + BlobDescriptor descriptor = new BlobDescriptor("/test/path", 10L, 20L); + BlobReference reference = new BlobReference(descriptor, "default.source", 7, 5L); + byte[] bytes = reference.serialize(); + bytes[0] = 2; + + assertThatThrownBy(() -> BlobReference.deserialize(bytes)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Expecting BlobReference version to be 1"); + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/types/InternalRowToSizeVisitorTest.java b/paimon-common/src/test/java/org/apache/paimon/types/InternalRowToSizeVisitorTest.java index cfdae649c190..728eb84e56e9 100644 --- a/paimon-common/src/test/java/org/apache/paimon/types/InternalRowToSizeVisitorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/types/InternalRowToSizeVisitorTest.java @@ -36,6 +36,11 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + /** Test for InternalRowToSizeVisitor. */ public class InternalRowToSizeVisitorTest { @@ -192,4 +197,18 @@ void testCalculatorSize() { Assertions.assertThat(feildSizeCalculator.get(23).apply(row, 23)).isEqualTo(0); } + + @Test + void testBlobRefSizeUsesSerializedReferenceBytes() { + DataGetters row = mock(DataGetters.class); + byte[] referenceBytes = new byte[] {1, 2, 3, 4}; + when(row.isNullAt(0)).thenReturn(false); + when(row.getBinary(0)).thenReturn(referenceBytes); + + int size = new InternalRowToSizeVisitor().visit(DataTypes.BLOB_REF()).apply(row, 0); + + Assertions.assertThat(size).isEqualTo(referenceBytes.length); + verify(row).getBinary(0); + verify(row, never()).getBlob(0); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 2ff1080c4a84..9a724b46a321 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -165,6 +165,7 @@ public static void validateTableSchema(TableSchema schema) { FileFormat fileFormat = FileFormat.fromIdentifier(options.formatType(), new Options(schema.options())); RowType tableRowType = new RowType(schema.fields()); + validateNestedBlobRefFields(tableRowType); Set blobDescriptorFields = validateBlobDescriptorFields(tableRowType, options); validateBlobExternalStorageFields(tableRowType, options, blobDescriptorFields); @@ -672,19 +673,22 @@ private static void validateRowTracking(TableSchema schema, CoreOptions options) List fields = schema.fields(); List blobNames = fields.stream() - .filter(field -> field.type().is(DataTypeRoot.BLOB)) + .filter( + field -> + field.type().is(DataTypeRoot.BLOB) + || field.type().is(DataTypeRoot.BLOB_REF)) .map(DataField::name) .collect(Collectors.toList()); if (!blobNames.isEmpty()) { checkArgument( options.dataEvolutionEnabled(), - "Data evolution config must enabled for table with BLOB type column."); + "Data evolution config must enabled for table with BLOB or BLOB_REF type column."); checkArgument( fields.size() > blobNames.size(), - "Table with BLOB type column must have other normal columns."); + "Table with BLOB or BLOB_REF type column must have other normal columns."); checkArgument( blobNames.stream().noneMatch(schema.partitionKeys()::contains), - "The BLOB type column can not be part of partition keys."); + "The BLOB or BLOB_REF type column can not be part of partition keys."); } FileFormat vectorFileFormat = vectorFileFormat(options); @@ -702,6 +706,49 @@ private static void validateRowTracking(TableSchema schema, CoreOptions options) } } + private static void validateNestedBlobRefFields(RowType rowType) { + for (DataField field : rowType.getFields()) { + checkArgument( + !containsNestedBlobRef(field.type()), + "Nested BLOB_REF type is not supported. Field '%s' contains a nested BLOB_REF.", + field.name()); + } + } + + private static boolean containsNestedBlobRef(DataType dataType) { + switch (dataType.getTypeRoot()) { + case ARRAY: + DataType arrayElementType = ((ArrayType) dataType).getElementType(); + return arrayElementType.is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(arrayElementType); + case MULTISET: + DataType multisetElementType = ((MultisetType) dataType).getElementType(); + return multisetElementType.is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(multisetElementType); + case MAP: + MapType mapType = (MapType) dataType; + return mapType.getKeyType().is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(mapType.getKeyType()) + || mapType.getValueType().is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(mapType.getValueType()); + case ROW: + for (DataField field : ((RowType) dataType).getFields()) { + if (field.type().is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(field.type())) { + return true; + } + } + return false; + case VECTOR: + DataType vectorElementType = + ((org.apache.paimon.types.VectorType) dataType).getElementType(); + return vectorElementType.is(DataTypeRoot.BLOB_REF) + || containsNestedBlobRef(vectorElementType); + default: + return false; + } + } + private static Set validateBlobDescriptorFields(RowType rowType, CoreOptions options) { Set blobFieldNames = rowType.getFields().stream() diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java index 327810b881bc..8cb70824cd45 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java @@ -123,7 +123,8 @@ public InnerTableRead newRead() { new AppendTableRawFileSplitReadProvider( () -> store().newRead(), config)); } - return new AppendTableRead(providerFactories, schema()); + return new AppendTableRead( + providerFactories, schema(), catalogEnvironment().catalogContext()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java index a2fee49bfb88..411bcf6767de 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java @@ -147,7 +147,10 @@ protected BiConsumer nonPartitionFilterConsumer() { @Override public InnerTableRead newRead() { return new KeyValueTableRead( - () -> store().newRead(), () -> store().newBatchRawFileRead(), schema()); + () -> store().newRead(), + () -> store().newBatchRawFileRead(), + schema(), + catalogEnvironment().catalogContext()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index ca5af88f40bc..0ed961487f80 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -18,16 +18,21 @@ package org.apache.paimon.table.source; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.BlobReferenceInternalRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BlobReferenceLookup; import org.apache.paimon.utils.ListUtils; import org.apache.paimon.utils.ProjectedRow; +import org.apache.paimon.utils.UriReaderFactory; import java.io.IOException; import java.util.ArrayList; @@ -45,9 +50,11 @@ public abstract class AbstractDataTableRead implements InnerTableRead { private boolean executeFilter = false; private Predicate predicate; private final TableSchema schema; + private final CatalogContext catalogContext; - public AbstractDataTableRead(TableSchema schema) { + public AbstractDataTableRead(TableSchema schema, CatalogContext catalogContext) { this.schema = schema; + this.catalogContext = catalogContext; } public abstract void applyReadType(RowType readType); @@ -106,6 +113,27 @@ public final RecordReader createReader(Split split) throws IOExcept reader = executeFilter(reader); } + if (catalogContext != null) { + RowType rowType = this.readType == null ? schema.logicalRowType() : this.readType; + int[] blobRefFields = + rowType.getFields().stream() + .filter(field -> field.type().is(DataTypeRoot.BLOB_REF)) + .mapToInt(field -> rowType.getFieldIndex(field.name())) + .toArray(); + if (blobRefFields.length > 0) { + UriReaderFactory uriReaderFactory = new UriReaderFactory(catalogContext); + reader = + reader.transform( + row -> + new BlobReferenceInternalRow( + row, + blobRefFields, + uriReaderFactory, + BlobReferenceLookup.createResolver( + catalogContext))); + } + } + return reader; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java index 1a9ed9b4bee2..388ef9344c1e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java @@ -18,6 +18,7 @@ package org.apache.paimon.table.source; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.InternalRow; import org.apache.paimon.operation.MergeFileSplitRead; import org.apache.paimon.operation.SplitRead; @@ -51,8 +52,9 @@ public final class AppendTableRead extends AbstractDataTableRead { public AppendTableRead( List> providerFactories, - TableSchema schema) { - super(schema); + TableSchema schema, + CatalogContext catalogContext) { + super(schema, catalogContext); this.readProviders = providerFactories.stream() .map(factory -> factory.apply(this::config)) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java index fda7d70ffdf6..ac83737afdc8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.KeyValue; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.operation.MergeFileSplitRead; @@ -63,8 +64,9 @@ public final class KeyValueTableRead extends AbstractDataTableRead { public KeyValueTableRead( Supplier mergeReadSupplier, Supplier batchRawReadSupplier, - TableSchema schema) { - super(schema); + TableSchema schema, + CatalogContext catalogContext) { + super(schema, catalogContext); this.readProviders = Arrays.asList( new PrimaryKeyTableRawFileSplitReadProvider( diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java new file mode 100644 index 000000000000..765c71970d05 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.BlobReferenceResolver; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; + +import java.util.Collections; + +/** Utilities for resolving {@link BlobReference} through table metadata. */ +public class BlobReferenceLookup { + + public static BlobReferenceResolver createResolver(CatalogContext catalogContext) { + return reference -> resolve(catalogContext, reference); + } + + public static Blob resolve(CatalogContext catalogContext, BlobReference reference) { + try (Catalog catalog = CatalogFactory.createCatalog(catalogContext)) { + Table table = catalog.getTable(Identifier.fromString(reference.tableName())); + if (!table.rowType().containsField(reference.fieldId())) { + throw new IllegalArgumentException( + "Cannot find blob fieldId " + + reference.fieldId() + + " in upstream table " + + reference.tableName() + + "."); + } + int fieldPos = table.rowType().getFieldIndexByFieldId(reference.fieldId()); + + ReadBuilder readBuilder = + table.newReadBuilder() + .withProjection(new int[] {fieldPos}) + .withRowRanges( + Collections.singletonList( + new Range(reference.rowId(), reference.rowId()))); + + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + RecordReader.RecordIterator batch; + while ((batch = reader.readBatch()) != null) { + try { + InternalRow row; + while ((row = batch.next()) != null) { + return row.getBlob(0); + } + } finally { + batch.releaseBatch(); + } + } + } + + throw new IllegalStateException( + "Cannot resolve blob reference for table " + + reference.tableName() + + ", rowId " + + reference.rowId() + + ", fieldId " + + reference.fieldId() + + "."); + } catch (Exception e) { + throw new RuntimeException("Failed to resolve blob reference fallback.", e); + } + } + + private BlobReferenceLookup() {} +} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/BlobRefSchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/BlobRefSchemaValidationTest.java new file mode 100644 index 000000000000..fcc345d719a9 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/schema/BlobRefSchemaValidationTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.schema; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static org.apache.paimon.CoreOptions.BUCKET; +import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for BLOB_REF-specific schema validation. */ +public class BlobRefSchemaValidationTest { + + @Test + public void testNestedBlobRefTableSchema() { + Map options = new HashMap<>(); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + options.put(BUCKET.key(), String.valueOf(-1)); + + List fields = + Arrays.asList( + new DataField(0, "f0", DataTypes.INT()), + new DataField( + 1, + "f1", + DataTypes.ROW(DataTypes.FIELD(2, "nested", DataTypes.BLOB_REF())))); + + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))) + .hasMessage( + "Nested BLOB_REF type is not supported. Field 'f1' contains a nested BLOB_REF."); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java new file mode 100644 index 000000000000..3a6dccb00854 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.metrics.MetricRegistry; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.OptionalLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests for {@link BlobReferenceLookup}. */ +public class BlobReferenceLookupTest { + + @Test + public void testResolveByFieldIdAfterRename() throws Exception { + CatalogContext context = mock(CatalogContext.class); + Catalog catalog = mock(Catalog.class); + Table table = mock(Table.class); + ReadBuilder readBuilder = mock(ReadBuilder.class); + TableScan scan = mock(TableScan.class); + TableScan.Plan plan = mock(TableScan.Plan.class); + + byte[] payload = new byte[] {1, 2, 3}; + InternalRow row = GenericRow.of(Blob.fromData(payload)); + Split split = new TestSplit(); + + when(catalog.getTable(any())).thenReturn(table); + when(table.rowType()) + .thenReturn( + new RowType( + Collections.singletonList( + new DataField(7, "blob_renamed", DataTypes.BLOB())))); + when(table.newReadBuilder()).thenReturn(readBuilder); + when(readBuilder.withProjection(any(int[].class))).thenReturn(readBuilder); + when(readBuilder.withRowRanges(anyList())).thenReturn(readBuilder); + when(readBuilder.newRead()).thenReturn(new SingleRowTableRead(split, row)); + when(readBuilder.newScan()).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.splits()).thenReturn(Collections.singletonList(split)); + + BlobReference reference = + new BlobReference( + new BlobDescriptor("file:/missing", 0L, -1L), "default.source", 7, 12L); + + try (MockedStatic mockedCatalogFactory = + Mockito.mockStatic(CatalogFactory.class)) { + mockedCatalogFactory + .when(() -> CatalogFactory.createCatalog(context)) + .thenReturn(catalog); + + Blob resolved = BlobReferenceLookup.resolve(context, reference); + + assertThat(resolved.toData()).isEqualTo(payload); + } + } + + private static class SingleRowTableRead implements TableRead { + + private final Split split; + private final InternalRow row; + + private SingleRowTableRead(Split split, InternalRow row) { + this.split = split; + this.row = row; + } + + @Override + public TableRead withMetricRegistry(MetricRegistry registry) { + return this; + } + + @Override + public TableRead executeFilter() { + return this; + } + + @Override + public TableRead withIOManager(IOManager ioManager) { + return this; + } + + @Override + public RecordReader createReader(Split split) { + assertThat(split).isSameAs(this.split); + return new RecordReader() { + + private boolean emitted = false; + + @Nullable + @Override + public RecordIterator readBatch() { + if (emitted) { + return null; + } + emitted = true; + return new RecordIterator() { + + private boolean rowReturned = false; + + @Nullable + @Override + public InternalRow next() { + if (rowReturned) { + return null; + } + rowReturned = true; + return row; + } + + @Override + public void releaseBatch() {} + }; + } + + @Override + public void close() throws IOException {} + }; + } + } + + private static class TestSplit implements Split { + + @Override + public long rowCount() { + return 1L; + } + + @Override + public OptionalLong mergedRowCount() { + return OptionalLong.of(1L); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java index 92ae714ca577..21cb45cf0d93 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java @@ -21,6 +21,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -155,6 +156,11 @@ public LogicalType visit(BlobType blobType) { org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); } + @Override + public LogicalType visit(BlobRefType blobRefType) { + return new org.apache.flink.table.types.logical.VarBinaryType(BlobType.DEFAULT_SIZE); + } + @Override public LogicalType visit(ArrayType arrayType) { return new org.apache.flink.table.types.logical.ArrayType( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java index 5f59063668a5..80d1d14d2d2f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java @@ -153,6 +153,7 @@ import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.catalog.Catalog.TOTAL_SIZE_PROP; import static org.apache.paimon.flink.FlinkCatalogOptions.DISABLE_CREATE_TABLE_IN_DEFAULT_DB; +import static org.apache.paimon.flink.LogicalTypeConversion.toBlobRefType; import static org.apache.paimon.flink.LogicalTypeConversion.toBlobType; import static org.apache.paimon.flink.LogicalTypeConversion.toDataType; import static org.apache.paimon.flink.LogicalTypeConversion.toLogicalType; @@ -1038,6 +1039,7 @@ public static Schema fromCatalogTable(CatalogBaseTable catalogTable) { Map options = new HashMap<>(catalogTable.getOptions()); List blobFields = CoreOptions.blobField(options); + List blobRefFields = CoreOptions.blobRefField(options); if (!blobFields.isEmpty()) { checkArgument( options.containsKey(CoreOptions.DATA_EVOLUTION_ENABLED.key()), @@ -1047,6 +1049,15 @@ public static Schema fromCatalogTable(CatalogBaseTable catalogTable) { + CoreOptions.DATA_EVOLUTION_ENABLED.key() + "'"); } + if (!blobRefFields.isEmpty()) { + checkArgument( + options.containsKey(CoreOptions.DATA_EVOLUTION_ENABLED.key()), + "When setting '" + + CoreOptions.BLOB_REF_FIELD.key() + + "', you must also set '" + + CoreOptions.DATA_EVOLUTION_ENABLED.key() + + "'"); + } // Serialize virtual columns and watermark to the options // This is what Flink SQL needs, the storage itself does not need them options.putAll(columnOptions(schema)); @@ -1077,9 +1088,13 @@ private static org.apache.paimon.types.DataType resolveDataType( org.apache.flink.table.types.logical.LogicalType logicalType, Map options) { List blobFields = CoreOptions.blobField(options); + List blobRefFields = CoreOptions.blobRefField(options); if (blobFields.contains(fieldName)) { return toBlobType(logicalType); } + if (blobRefFields.contains(fieldName)) { + return toBlobRefType(logicalType); + } Set vectorFields = CoreOptions.vectorField(options); if (vectorFields.contains(fieldName)) { return toVectorType(fieldName, logicalType, options); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowWrapper.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowWrapper.java index ad2132e8c1eb..2b652f6d686c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowWrapper.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowWrapper.java @@ -22,7 +22,8 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobData; -import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobReferenceResolver; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; @@ -32,7 +33,7 @@ import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.types.RowKind; -import org.apache.paimon.utils.UriReader; +import org.apache.paimon.utils.BlobReferenceLookup; import org.apache.paimon.utils.UriReaderFactory; import org.apache.flink.table.data.DecimalData; @@ -48,6 +49,7 @@ public class FlinkRowWrapper implements InternalRow { private final org.apache.flink.table.data.RowData row; private final UriReaderFactory uriReaderFactory; + private final BlobReferenceResolver blobReferenceResolver; public FlinkRowWrapper(org.apache.flink.table.data.RowData row) { this(row, null); @@ -55,7 +57,10 @@ public FlinkRowWrapper(org.apache.flink.table.data.RowData row) { public FlinkRowWrapper(org.apache.flink.table.data.RowData row, CatalogContext catalogContext) { this.row = row; - this.uriReaderFactory = new UriReaderFactory(catalogContext); + this.uriReaderFactory = + catalogContext == null ? null : new UriReaderFactory(catalogContext); + this.blobReferenceResolver = + catalogContext == null ? null : BlobReferenceLookup.createResolver(catalogContext); } @Override @@ -142,15 +147,8 @@ public Variant getVariant(int pos) { @Override public Blob getBlob(int pos) { - byte[] bytes = row.getBinary(pos); - boolean blobDes = BlobDescriptor.isBlobDescriptor(bytes); - if (blobDes) { - BlobDescriptor blobDescriptor = BlobDescriptor.deserialize(bytes); - UriReader uriReader = uriReaderFactory.create(blobDescriptor.uri()); - return Blob.fromDescriptor(uriReader, blobDescriptor); - } else { - return new BlobData(bytes); - } + return BlobUtils.fromBytes( + row.getBinary(pos), uriReaderFactory, blobReferenceResolver, null); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/LogicalTypeConversion.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/LogicalTypeConversion.java index 556dbd95ff31..1cd0168c332d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/LogicalTypeConversion.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/LogicalTypeConversion.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink; import org.apache.paimon.CoreOptions; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; @@ -53,6 +54,13 @@ public static BlobType toBlobType(LogicalType logicalType) { return new BlobType(); } + public static BlobRefType toBlobRefType(LogicalType logicalType) { + checkArgument( + logicalType instanceof BinaryType || logicalType instanceof VarBinaryType, + "Expected BinaryType or VarBinaryType, but got: " + logicalType); + return new BlobRefType(); + } + public static VectorType toVectorType( String fieldName, org.apache.flink.table.types.logical.LogicalType logicalType, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java index e4870de58336..76d83393b3d4 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink.lookup; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.operation.MergeFileSplitRead; @@ -41,8 +42,9 @@ public class LookupCompactDiffRead extends AbstractDataTableRead { private final SplitRead fullPhaseMergeRead; private final SplitRead incrementalDiffRead; - public LookupCompactDiffRead(MergeFileSplitRead mergeRead, TableSchema schema) { - super(schema); + public LookupCompactDiffRead( + MergeFileSplitRead mergeRead, TableSchema schema, CatalogContext catalogContext) { + super(schema, catalogContext); this.incrementalDiffRead = new IncrementalCompactDiffSplitRead(mergeRead); this.fullPhaseMergeRead = SplitRead.convert( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupFileStoreTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupFileStoreTable.java index 353c99d2b1f1..4e355db1e8f2 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupFileStoreTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupFileStoreTable.java @@ -64,7 +64,9 @@ public InnerTableRead newRead() { return wrapped.newRead(); case COMPACT_DELTA_MONITOR: return new LookupCompactDiffRead( - ((KeyValueFileStore) wrapped.store()).newRead(), wrapped.schema()); + ((KeyValueFileStore) wrapped.store()).newRead(), + wrapped.schema(), + wrapped.catalogEnvironment().catalogContext()); default: throw new UnsupportedOperationException( "Unknown lookup stream scan mode: " + lookupScanMode.name()); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java index b49b9adb9476..eff22090735c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceSplitReader.java @@ -277,7 +277,8 @@ private FileStoreRecordIterator(@Nullable RowType rowType) { private Set blobFieldIndex(RowType rowType) { Set result = new HashSet<>(); for (int i = 0; i < rowType.getFieldCount(); i++) { - if (rowType.getTypeAt(i).getTypeRoot() == DataTypeRoot.BLOB) { + if (rowType.getTypeAt(i).getTypeRoot() == DataTypeRoot.BLOB + || rowType.getTypeAt(i).getTypeRoot() == DataTypeRoot.BLOB_REF) { result.add(i); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java index 8114ac17eb38..a0d5f98ba0e0 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java @@ -152,7 +152,7 @@ public KeyValueTableRead createReadWithKey() { FileFormatDiscover.of(options), pathFactory, options); - return new KeyValueTableRead(() -> read, () -> rawFileRead, null); + return new KeyValueTableRead(() -> read, () -> rawFileRead, schema, null); } public List writeFiles( diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaConverter.java index eb55a86c5b66..251a973b3e0c 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaConverter.java @@ -106,6 +106,7 @@ public static Schema convertToSchema( case BINARY: case VARBINARY: case BLOB: + case BLOB_REF: Schema binary = SchemaBuilder.builder().bytesType(); return nullable ? nullableSchema(binary) : binary; case TIMESTAMP_WITHOUT_TIME_ZONE: diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java index 9aa663df8946..47f12d45684b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java @@ -89,10 +89,13 @@ public FieldReaderFactory(@Nullable UriReader uriReader) { @Override public FieldReader primitive(Schema primitive, DataType type) { - if (primitive.getType() == Schema.Type.BYTES - && type != null - && type.getTypeRoot() == DataTypeRoot.BLOB) { - return new BlobDescriptorBytesReader(uriReader); + if (primitive.getType() == Schema.Type.BYTES && type != null) { + if (type.getTypeRoot() == DataTypeRoot.BLOB) { + return new BlobDescriptorBytesReader(uriReader); + } + if (type.getTypeRoot() == DataTypeRoot.BLOB_REF) { + return BYTES_READER; + } } return AvroSchemaVisitor.super.primitive(primitive, type); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldWriterFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldWriterFactory.java index 6eb81cb7f5d1..d48becb7ed46 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldWriterFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldWriterFactory.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.DataGetters; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.GenericRow; @@ -93,12 +94,24 @@ public class FieldWriterFactory implements AvroSchemaVisitor { } }; + private static final FieldWriter BLOB_REFERENCE_BYTES_WRITER = + (container, i, encoder) -> { + Blob blob = container.getBlob(i); + if (blob == null) { + throw new IllegalArgumentException("Null blob_ref is not allowed."); + } + encoder.writeBytes(BlobUtils.serializeBlobReference(blob)); + }; + @Override public FieldWriter primitive(Schema primitive, DataType type) { - if (primitive.getType() == Schema.Type.BYTES - && type != null - && type.getTypeRoot() == DataTypeRoot.BLOB) { - return BLOB_DESCRIPTOR_BYTES_WRITER; + if (primitive.getType() == Schema.Type.BYTES && type != null) { + if (type.getTypeRoot() == DataTypeRoot.BLOB) { + return BLOB_DESCRIPTOR_BYTES_WRITER; + } + if (type.getTypeRoot() == DataTypeRoot.BLOB_REF) { + return BLOB_REFERENCE_BYTES_WRITER; + } } return AvroSchemaVisitor.super.primitive(primitive, type); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java index 4b80827d1bb3..4fe8b5999b3f 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java @@ -68,6 +68,7 @@ static TypeDescription convertToOrcType(DataType type, int fieldId, int depth) { return TypeDescription.createBoolean() .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); case BLOB: + case BLOB_REF: return TypeDescription.createBinary() .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); case VARBINARY: diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/FieldWriterFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/FieldWriterFactory.java index 443c2410cbd2..8248937b5f14 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/FieldWriterFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/FieldWriterFactory.java @@ -28,6 +28,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -264,6 +265,18 @@ public FieldWriter visit(BlobType blobType) { }; } + @Override + public FieldWriter visit(BlobRefType blobRefType) { + return (rowId, column, getters, columnId) -> { + BytesColumnVector vector = (BytesColumnVector) column; + byte[] bytes = + org.apache.paimon.data.BlobUtils.serializeBlobReference( + getters.getBlob(columnId)); + vector.setVal(rowId, bytes, 0, bytes.length); + return bytes.length; + }; + } + @Override public FieldWriter visit(DecimalType decimalType) { return (rowId, column, getters, columnId) -> { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 37a69fe9aebd..102aa0b2b709 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -91,6 +91,7 @@ public static Type convertToParquetType(String name, DataType type, int fieldId, case BINARY: case VARBINARY: case BLOB: + case BLOB_REF: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .named(name) .withId(fieldId); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java index a2741f869ab6..a285491219a6 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java @@ -101,6 +101,7 @@ public static WritableColumnVector createWritableColumnVector( case VARCHAR: case VARBINARY: case BLOB: + case BLOB_REF: return new HeapBytesVector(batchSize); case BINARY: return new HeapBytesVector(batchSize); @@ -178,7 +179,8 @@ public static ColumnVector createReadableColumnVector( case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return new ParquetTimestampVector(writableVector); case BLOB: - // Physical representation is bytes; higher-level Row#getBlob() handles descriptor. + case BLOB_REF: + // Physical representation is bytes; higher-level Row#getBlob() materializes them. return writableVector; case ARRAY: return new CastedArrayColumnVector( diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java index 0abf78fd2747..2f2582b401e6 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java @@ -36,6 +36,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -230,6 +231,11 @@ public UpdaterFactory visit(BlobType blobType) { }; } + @Override + public UpdaterFactory visit(BlobRefType blobRefType) { + return visit(new BlobType(blobRefType.isNullable())); + } + @Override public UpdaterFactory visit(ArrayType arrayType) { throw new RuntimeException("Array type is not supported"); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java index a7241147e68a..b668a0d912c9 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; @@ -109,6 +110,8 @@ private FieldWriter createWriter(DataType t, Type type) { return new BinaryWriter(); case BLOB: return new BlobDescriptorWriter(); + case BLOB_REF: + return new BlobReferenceWriter(); case DECIMAL: DecimalType decimalType = (DecimalType) t; return createDecimalWriter(decimalType.getPrecision(), decimalType.getScale()); @@ -344,6 +347,25 @@ private void writeBlob(Blob blob) { } } + /** Writes BLOB_REF as serialized {@link org.apache.paimon.data.BlobReference} bytes. */ + private class BlobReferenceWriter implements FieldWriter { + + @Override + public void write(InternalRow row, int ordinal) { + writeBlob(row.getBlob(ordinal)); + } + + @Override + public void write(InternalArray arrayData, int ordinal) { + throw new UnsupportedOperationException("BLOB_REF in array is not supported."); + } + + private void writeBlob(Blob blob) { + byte[] bytes = BlobUtils.serializeBlobReference(blob); + recordConsumer.addBinary(Binary.fromReusedByteArray(bytes)); + } + } + private class IntWriter implements FieldWriter { @Override diff --git a/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java b/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java index dacd12f492c1..70c865f0b5f9 100644 --- a/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java +++ b/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java @@ -29,6 +29,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -414,6 +415,11 @@ public Operators.Column visit(BlobType blobType) { throw new UnsupportedOperationException(); } + @Override + public Operators.Column visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException(); + } + // ===================== can not support ========================= @Override diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index cb3d7de27da5..10e403eb154a 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -18,6 +18,9 @@ package org.apache.paimon.format.avro; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobReference; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -76,6 +79,7 @@ public void testSupportedDataTypes() { dataFields.add(new DataField(index++, "varchar_type", DataTypes.VARCHAR(20))); dataFields.add(new DataField(index++, "binary_type", DataTypes.BINARY(20))); dataFields.add(new DataField(index++, "varbinary_type", DataTypes.VARBINARY(20))); + dataFields.add(new DataField(index++, "blob_ref_type", DataTypes.BLOB_REF())); dataFields.add(new DataField(index++, "timestamp_type", DataTypes.TIMESTAMP(3))); dataFields.add(new DataField(index++, "date_type", DataTypes.DATE())); dataFields.add(new DataField(index++, "decimal_type", DataTypes.DECIMAL(10, 3))); @@ -210,4 +214,34 @@ void testCompression() throws IOException { .hasMessageContaining("Unrecognized codec: unsupported"); } } + + @Test + void testBlobRefRoundTrip() throws IOException { + RowType rowType = DataTypes.ROW(DataTypes.FIELD(0, "blob_ref", DataTypes.BLOB_REF())); + BlobReference reference = + new BlobReference( + new BlobDescriptor("file:/tmp/blob-ref", 12L, 34L), + "default.t", + 7, + 11L); + Blob blob = Blob.fromReference(null, null, reference); + + FileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(blob)); + writer.close(); + } + + try (RecordReader reader = + format.createReaderFactory(rowType, rowType, new ArrayList<>()) + .createReader( + new FormatReaderContext(fileIO, file, fileIO.getFileSize(file)))) { + InternalRow row = reader.readBatch().next(); + assertThat(row.getBinary(0)).isEqualTo(reference.serialize()); + } + } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java index 5669ac33d443..5c36e14cfd1a 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java @@ -60,6 +60,7 @@ void testDataTypeToOrcType() { test("varchar(123)", DataTypes.VARCHAR(123)); test("string", DataTypes.STRING()); test("binary", DataTypes.BYTES()); + test("binary", DataTypes.BLOB_REF()); test("tinyint", DataTypes.TINYINT()); test("smallint", DataTypes.SMALLINT()); test("int", DataTypes.INT()); diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index bfbdaed7c4a3..fc308c11c518 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.types.RowType; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -106,4 +107,16 @@ public void testPaimonParquetSchemaConvert() { RowType rowType = convertToPaimonRowType(messageType); assertThat(ALL_TYPES).isEqualTo(rowType); } + + @Test + public void testBlobRefSchemaConvertToBinary() { + MessageType messageType = + convertToParquetMessageType( + new RowType( + Arrays.asList( + new DataField(0, "blob_ref", DataTypes.BLOB_REF())))); + + assertThat(messageType.getType("blob_ref").asPrimitiveType().getPrimitiveTypeName()) + .isEqualTo(PrimitiveType.PrimitiveTypeName.BINARY); + } } diff --git a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java index e4799341d1dc..bcb48dffb485 100644 --- a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java +++ b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java @@ -22,6 +22,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -235,6 +236,11 @@ public TypeInfo visit(BlobType blobType) { return TypeInfoFactory.binaryTypeInfo; } + @Override + public TypeInfo visit(BlobRefType blobRefType) { + return TypeInfoFactory.binaryTypeInfo; + } + @Override protected TypeInfo defaultMethod(org.apache.paimon.types.DataType dataType) { throw new UnsupportedOperationException("Unsupported type: " + dataType); diff --git a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java index 64b4e2887f82..9421de6b60a3 100644 --- a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java +++ b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java @@ -27,6 +27,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -188,6 +189,11 @@ public Void visit(BlobType blobType) { return null; } + @Override + public Void visit(BlobRefType blobRefType) { + return null; + } + @Override public Void visit(ArrayType arrayType) { return null; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 6ef853eda870..7a07f4fb5ef8 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -41,6 +41,7 @@ import org.apache.paimon.table.iceberg.IcebergTable; import org.apache.paimon.table.lance.LanceTable; import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -457,6 +458,7 @@ private Schema toInitialSchema( StructType schema, Transform[] partitions, Map properties) { Map normalizedProperties = new HashMap<>(properties); List blobFields = CoreOptions.blobField(properties); + List blobRefFields = CoreOptions.blobRefField(properties); String provider = properties.get(TableCatalog.PROP_PROVIDER); if (!usePaimon(provider)) { if (isFormatTable(provider)) { @@ -495,6 +497,11 @@ private Schema toInitialSchema( field.dataType() instanceof org.apache.spark.sql.types.BinaryType, "The type of blob field must be binary"); type = new BlobType(); + } else if (blobRefFields.contains(name)) { + checkArgument( + field.dataType() instanceof org.apache.spark.sql.types.BinaryType, + "The type of blob ref field must be binary"); + type = new BlobRefType(); } else { type = toPaimonType(field.dataType()).copy(field.nullable()); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java index ffd077741c9f..23195a83f5b6 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java @@ -22,7 +22,8 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobData; -import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobReferenceResolver; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; @@ -32,7 +33,7 @@ import org.apache.paimon.data.variant.Variant; import org.apache.paimon.spark.util.shim.TypeUtils$; import org.apache.paimon.types.RowKind; -import org.apache.paimon.utils.UriReader; +import org.apache.paimon.utils.BlobReferenceLookup; import org.apache.paimon.utils.UriReaderFactory; import org.apache.spark.sql.catalyst.util.ArrayData; @@ -62,6 +63,7 @@ public class SparkInternalRowWrapper implements InternalRow, Serializable { private final StructType tableSchema; private final int length; @Nullable private final UriReaderFactory uriReaderFactory; + @Nullable private final BlobReferenceResolver blobReferenceResolver; @Nullable private final int[] fieldIndexMap; @Nullable private final StructType dataSchema; @@ -82,6 +84,8 @@ public SparkInternalRowWrapper( this.fieldIndexMap = dataSchema != null ? buildFieldIndexMap(tableSchema, dataSchema) : null; this.uriReaderFactory = new UriReaderFactory(catalogContext); + this.blobReferenceResolver = + catalogContext == null ? null : BlobReferenceLookup.createResolver(catalogContext); } public SparkInternalRowWrapper replace(org.apache.spark.sql.catalyst.InternalRow internalRow) { @@ -246,15 +250,8 @@ public Blob getBlob(int pos) { if (actualPos == -1 || internalRow.isNullAt(actualPos)) { return null; } - byte[] bytes = internalRow.getBinary(actualPos); - boolean blobDes = BlobDescriptor.isBlobDescriptor(bytes); - if (blobDes) { - BlobDescriptor blobDescriptor = BlobDescriptor.deserialize(bytes); - UriReader uriReader = uriReaderFactory.create(blobDescriptor.uri()); - return Blob.fromDescriptor(uriReader, blobDescriptor); - } else { - return new BlobData(bytes); - } + return BlobUtils.fromBytes( + internalRow.getBinary(actualPos), uriReaderFactory, blobReferenceResolver, null); } @Override diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java index 36b5624ff52f..1fc04c817844 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java @@ -22,7 +22,8 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobData; -import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobReferenceResolver; +import org.apache.paimon.data.BlobUtils; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; @@ -37,8 +38,8 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BlobReferenceLookup; import org.apache.paimon.utils.DateTimeUtils; -import org.apache.paimon.utils.UriReader; import org.apache.paimon.utils.UriReaderFactory; import org.apache.spark.sql.Row; @@ -63,6 +64,7 @@ public class SparkRow implements InternalRow, Serializable { private final Row row; private final RowKind rowKind; private final UriReaderFactory uriReaderFactory; + private final BlobReferenceResolver blobReferenceResolver; public SparkRow(RowType type, Row row) { this(type, row, RowKind.INSERT, null); @@ -72,7 +74,10 @@ public SparkRow(RowType type, Row row, RowKind rowkind, CatalogContext catalogCo this.type = type; this.row = row; this.rowKind = rowkind; - this.uriReaderFactory = new UriReaderFactory(catalogContext); + this.uriReaderFactory = + catalogContext == null ? null : new UriReaderFactory(catalogContext); + this.blobReferenceResolver = + catalogContext == null ? null : BlobReferenceLookup.createResolver(catalogContext); } @Override @@ -161,15 +166,7 @@ public Variant getVariant(int i) { @Override public Blob getBlob(int i) { - byte[] bytes = row.getAs(i); - boolean blobDes = BlobDescriptor.isBlobDescriptor(bytes); - if (blobDes) { - BlobDescriptor blobDescriptor = BlobDescriptor.deserialize(bytes); - UriReader uriReader = uriReaderFactory.create(blobDescriptor.uri()); - return Blob.fromDescriptor(uriReader, blobDescriptor); - } else { - return new BlobData(bytes); - } + return BlobUtils.fromBytes(row.getAs(i), uriReaderFactory, blobReferenceResolver, null); } @Override diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java index dc2f8b30acab..823534deea7c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; import org.apache.paimon.types.BlobType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; @@ -167,6 +168,11 @@ public DataType visit(BlobType blobType) { return DataTypes.BinaryType; } + @Override + public DataType visit(BlobRefType blobRefType) { + return DataTypes.BinaryType; + } + @Override public DataType visit(VarBinaryType varBinaryType) { return DataTypes.BinaryType; diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala index ae504b24120f..75c1bd1337ff 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala @@ -48,7 +48,8 @@ object SparkInternalRow { var i: Int = 0 val blobFields = new mutable.HashSet[Int]() while (i < rowType.getFieldCount) { - if (rowType.getTypeAt(i).getTypeRoot.equals(DataTypeRoot.BLOB)) { + if (rowType.getTypeAt(i).getTypeRoot.equals(DataTypeRoot.BLOB) || + rowType.getTypeAt(i).getTypeRoot.equals(DataTypeRoot.BLOB_REF)) { blobFields.add(i) } i += 1 diff --git a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormat.java b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormat.java index eda8a3944ca9..d6191af2204d 100644 --- a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormat.java +++ b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormat.java @@ -27,6 +27,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobRefType; import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; @@ -187,6 +188,12 @@ public Void visit(BlobType blobType) { "Vortex file format does not support type BLOB"); } + @Override + public Void visit(BlobRefType blobRefType) { + throw new UnsupportedOperationException( + "Vortex file format does not support type BLOB_REF"); + } + @Override public Void visit(ArrayType arrayType) { return null; From 3997aefa278be6c3dd1394cc7ecd807b9d25fe3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Fri, 10 Apr 2026 15:22:27 +0800 Subject: [PATCH 2/3] Kiro 01 --- .../java/org/apache/paimon/data/Blob.java | 8 +- .../org/apache/paimon/data/BlobReference.java | 50 ++--- .../apache/paimon/data/BlobReferenceBlob.java | 45 ++-- .../paimon/data/BlobReferenceInternalRow.java | 13 +- .../org/apache/paimon/data/BlobUtils.java | 14 +- .../data/serializer/BlobRefSerializer.java | 2 +- .../paimon/data/BlobReferenceBlobTest.java | 6 +- .../apache/paimon/data/BlobReferenceTest.java | 30 ++- .../paimon/io/BlobReferenceLookupFile.java | 91 ++++++++ .../apache/paimon/io/RowDataFileWriter.java | 62 +++++- .../table/AppendOnlyFileStoreTable.java | 2 +- .../table/PrimaryKeyFileStoreTable.java | 3 +- .../table/source/AbstractDataTableRead.java | 14 +- .../paimon/table/source/AppendTableRead.java | 6 +- .../table/source/KeyValueTableRead.java | 6 +- .../paimon/utils/BlobReferenceLookup.java | 194 +++++++++++++++++- .../paimon/io/RowDataFileWriterTest.java | 95 +++++++++ .../paimon/utils/BlobReferenceLookupTest.java | 112 ++++++++-- .../flink/lookup/LookupCompactDiffRead.java | 2 +- .../source/TestChangelogDataReadWrite.java | 2 +- .../format/avro/AvroFileFormatTest.java | 4 +- 21 files changed, 649 insertions(+), 112 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/BlobReferenceLookupFile.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/io/RowDataFileWriterTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java index 88733d884d45..ba3bc4b02224 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java @@ -23,7 +23,6 @@ import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.utils.UriReader; -import org.apache.paimon.utils.UriReaderFactory; import java.io.IOException; import java.util.function.Supplier; @@ -66,11 +65,8 @@ static Blob fromDescriptor(UriReader reader, BlobDescriptor descriptor) { return new BlobRef(reader, descriptor); } - static Blob fromReference( - UriReaderFactory uriReaderFactory, - BlobReferenceResolver fallbackResolver, - BlobReference reference) { - return new BlobReferenceBlob(uriReaderFactory, fallbackResolver, reference); + static Blob fromReference(BlobReferenceResolver resolver, BlobReference reference) { + return new BlobReferenceBlob(resolver, reference); } static Blob fromInputStream(Supplier supplier) { diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java index 8e259e0ef895..d4a7926bd514 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReference.java @@ -25,7 +25,26 @@ import static java.nio.charset.StandardCharsets.UTF_8; -/** Serialized metadata for a {@code BLOB_REF} field. */ +/** + * Serialized metadata for a {@code BLOB_REF} field. + * + *

A blob reference only stores the coordinates needed to locate the original blob value in the + * upstream table: {@code tableName}, {@code fieldId} and {@code rowId}. The actual blob data is + * resolved at read time by scanning the upstream table. + * + *

Serialization layout (Little Endian): + * + * 

+ * | Offset       | Field         | Type    | Size |
+ * |--------------|---------------|---------|------|
+ * | 0            | version       | byte    | 1    |
+ * | 1            | magicNumber   | long    | 8    |
+ * | 9            | tableNameLen  | int     | 4    |
+ * | 13           | tableNameBytes| byte[N] | N    |
+ * | 13 + N       | fieldId       | int     | 4    |
+ * | 17 + N       | rowId         | long    | 8    |
+ *
+ */ public class BlobReference implements Serializable { private static final long serialVersionUID = 1L; @@ -33,22 +52,16 @@ public class BlobReference implements Serializable { private static final long MAGIC = 0x424C4F4252454631L; // "BLOBREF1" private static final byte CURRENT_VERSION = 1; - private final BlobDescriptor descriptor; private final String tableName; private final int fieldId; private final long rowId; - public BlobReference(BlobDescriptor descriptor, String tableName, int fieldId, long rowId) { - this.descriptor = descriptor; + public BlobReference(String tableName, int fieldId, long rowId) { this.tableName = tableName; this.fieldId = fieldId; this.rowId = rowId; } - public BlobDescriptor descriptor() { - return descriptor; - } - public String tableName() { return tableName; } @@ -62,15 +75,12 @@ public long rowId() { } public byte[] serialize() { - byte[] descriptorBytes = descriptor.serialize(); byte[] tableBytes = tableName.getBytes(UTF_8); - int totalSize = 1 + 8 + 4 + descriptorBytes.length + 4 + tableBytes.length + 4 + 8; + int totalSize = 1 + 8 + 4 + tableBytes.length + 4 + 8; ByteBuffer buffer = ByteBuffer.allocate(totalSize).order(ByteOrder.LITTLE_ENDIAN); buffer.put(CURRENT_VERSION); buffer.putLong(MAGIC); - buffer.putInt(descriptorBytes.length); - buffer.put(descriptorBytes); buffer.putInt(tableBytes.length); buffer.put(tableBytes); buffer.putInt(fieldId); @@ -81,6 +91,7 @@ public byte[] serialize() { public static BlobReference deserialize(byte[] bytes) { ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); byte version = buffer.get(); + if (version != CURRENT_VERSION) { throw new UnsupportedOperationException( "Expecting BlobReference version to be " @@ -99,19 +110,12 @@ public static BlobReference deserialize(byte[] bytes) { + magic); } - byte[] descriptorBytes = new byte[buffer.getInt()]; - buffer.get(descriptorBytes); - byte[] tableBytes = new byte[buffer.getInt()]; buffer.get(tableBytes); int fieldId = buffer.getInt(); long rowId = buffer.getLong(); - return new BlobReference( - BlobDescriptor.deserialize(descriptorBytes), - new String(tableBytes, UTF_8), - fieldId, - rowId); + return new BlobReference(new String(tableBytes, UTF_8), fieldId, rowId); } public static boolean isBlobReference(byte[] bytes) { @@ -119,7 +123,8 @@ public static boolean isBlobReference(byte[] bytes) { return false; } ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); - return buffer.get() == CURRENT_VERSION && MAGIC == buffer.getLong(); + byte version = buffer.get(); + return version == CURRENT_VERSION && MAGIC == buffer.getLong(); } @Override @@ -130,12 +135,11 @@ public boolean equals(Object o) { BlobReference that = (BlobReference) o; return fieldId == that.fieldId && rowId == that.rowId - && Objects.equals(descriptor, that.descriptor) && Objects.equals(tableName, that.tableName); } @Override public int hashCode() { - return Objects.hash(descriptor, tableName, fieldId, rowId); + return Objects.hash(tableName, fieldId, rowId); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java index b42cd10a0b4e..2f3a43cbb6f6 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceBlob.java @@ -20,7 +20,6 @@ import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.utils.IOUtils; -import org.apache.paimon.utils.UriReaderFactory; import javax.annotation.Nullable; @@ -28,21 +27,23 @@ import java.io.Serializable; import java.util.Objects; -/** {@link Blob} implementation backed by a serialized {@link BlobReference}. */ +/** + * {@link Blob} implementation backed by a serialized {@link BlobReference}. + * + *

The actual blob data is resolved lazily through the {@link BlobReferenceResolver} which scans + * the upstream table using the coordinates stored in the reference. + */ public class BlobReferenceBlob implements Blob, Serializable { private static final long serialVersionUID = 1L; - private final @Nullable BlobReferenceResolver fallbackResolver; - private final @Nullable UriReaderFactory uriReaderFactory; + private final @Nullable BlobReferenceResolver resolver; private final BlobReference reference; + private transient @Nullable Blob resolvedBlob; public BlobReferenceBlob( - @Nullable UriReaderFactory uriReaderFactory, - @Nullable BlobReferenceResolver fallbackResolver, - BlobReference reference) { - this.uriReaderFactory = uriReaderFactory; - this.fallbackResolver = fallbackResolver; + @Nullable BlobReferenceResolver resolver, BlobReference reference) { + this.resolver = resolver; this.reference = reference; } @@ -61,25 +62,25 @@ public byte[] toData() { @Override public BlobDescriptor toDescriptor() { - return reference.descriptor(); + return resolve().toDescriptor(); } @Override public SeekableInputStream newInputStream() throws IOException { - if (uriReaderFactory != null) { - try { - return Blob.fromDescriptor( - uriReaderFactory.create(reference.descriptor().uri()), - reference.descriptor()) - .newInputStream(); - } catch (Exception ignored) { - // Fall through to the metadata-based lookup. - } + return resolve().newInputStream(); + } + + private Blob resolve() { + if (resolvedBlob != null) { + return resolvedBlob; } - if (fallbackResolver == null) { - throw new IOException("Blob reference cannot be resolved without fallback resolver."); + + if (resolver == null) { + throw new IllegalStateException( + "Blob reference cannot be resolved without a resolver."); } - return fallbackResolver.resolve(reference).newInputStream(); + resolvedBlob = resolver.resolve(reference); + return resolvedBlob; } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java index 1f3e1a14c551..b8663eb10650 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobReferenceInternalRow.java @@ -19,7 +19,6 @@ package org.apache.paimon.data; import org.apache.paimon.types.RowKind; -import org.apache.paimon.utils.UriReaderFactory; import java.io.Serializable; import java.util.HashSet; @@ -32,21 +31,18 @@ public class BlobReferenceInternalRow implements InternalRow, Serializable { private final InternalRow wrapped; private final Set blobRefFields; - private final UriReaderFactory uriReaderFactory; - private final BlobReferenceResolver fallbackResolver; + private final BlobReferenceResolver resolver; public BlobReferenceInternalRow( InternalRow wrapped, int[] blobRefFields, - UriReaderFactory uriReaderFactory, - BlobReferenceResolver fallbackResolver) { + BlobReferenceResolver resolver) { this.wrapped = wrapped; this.blobRefFields = new HashSet<>(); for (int field : blobRefFields) { this.blobRefFields.add(field); } - this.uriReaderFactory = uriReaderFactory; - this.fallbackResolver = fallbackResolver; + this.resolver = resolver; } @Override @@ -134,8 +130,7 @@ public Blob getBlob(int pos) { if (!blobRefFields.contains(pos)) { return wrapped.getBlob(pos); } - return BlobUtils.fromBytes( - wrapped.getBinary(pos), uriReaderFactory, fallbackResolver, null); + return BlobUtils.fromBytes(wrapped.getBinary(pos), null, resolver, null); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java b/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java index 8acf8ee669a7..532d83eb8169 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobUtils.java @@ -30,15 +30,19 @@ public class BlobUtils { public static Blob fromBytes( byte[] bytes, @Nullable UriReaderFactory uriReaderFactory, - @Nullable BlobReferenceResolver fallbackResolver, + @Nullable BlobReferenceResolver resolver, @Nullable FileIO fileIO) { + if (bytes == null) { + return null; + } + if (BlobReference.isBlobReference(bytes)) { - if (uriReaderFactory == null || fallbackResolver == null) { + BlobReference reference = BlobReference.deserialize(bytes); + if (resolver == null) { throw new IllegalStateException( - "Blob reference bytes require both uri reader factory and fallback resolver."); + "Blob reference bytes require a resolver to look up the upstream table."); } - return new BlobReferenceBlob( - uriReaderFactory, fallbackResolver, BlobReference.deserialize(bytes)); + return new BlobReferenceBlob(resolver, reference); } if (BlobDescriptor.isBlobDescriptor(bytes)) { diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java index 03c304ed1cce..5bf37225859d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BlobRefSerializer.java @@ -47,6 +47,6 @@ public void serialize(Blob blob, DataOutputView target) throws IOException { @Override public Blob deserialize(DataInputView source) throws IOException { byte[] bytes = BinarySerializer.INSTANCE.deserialize(source); - return new BlobReferenceBlob(null, null, BlobReference.deserialize(bytes)); + return new BlobReferenceBlob(null, BlobReference.deserialize(bytes)); } } diff --git a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java index c57d9509e791..09d0c9effbf0 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceBlobTest.java @@ -33,9 +33,7 @@ public class BlobReferenceBlobTest { @Test public void testFallbackKeepsStreamingInput() throws Exception { - BlobReference reference = - new BlobReference( - new BlobDescriptor("file:/missing", 0L, -1L), "default.source", 7, 5L); + BlobReference reference = new BlobReference("default.source", 7, 5L); BlobReferenceResolver resolver = mock(BlobReferenceResolver.class); Blob resolved = mock(Blob.class); SeekableInputStream inputStream = mock(SeekableInputStream.class); @@ -43,7 +41,7 @@ public void testFallbackKeepsStreamingInput() throws Exception { when(resolved.newInputStream()).thenReturn(inputStream); SeekableInputStream actual = - new BlobReferenceBlob(null, resolver, reference).newInputStream(); + new BlobReferenceBlob(resolver, reference).newInputStream(); assertThat(actual).isSameAs(inputStream); verify(resolver).resolve(reference); diff --git a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java index 0ba357b7e009..7bbb7965ba95 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobReferenceTest.java @@ -28,12 +28,10 @@ public class BlobReferenceTest { @Test public void testSerializeAndDeserialize() { - BlobDescriptor descriptor = new BlobDescriptor("/test/path", 10L, 20L); - BlobReference reference = new BlobReference(descriptor, "default.source", 7, 5L); + BlobReference reference = new BlobReference("default.source", 7, 5L); BlobReference deserialized = BlobReference.deserialize(reference.serialize()); - assertThat(deserialized.descriptor()).isEqualTo(descriptor); assertThat(deserialized.tableName()).isEqualTo("default.source"); assertThat(deserialized.fieldId()).isEqualTo(7); assertThat(deserialized.rowId()).isEqualTo(5L); @@ -41,13 +39,33 @@ public void testSerializeAndDeserialize() { @Test public void testRejectUnexpectedVersion() { - BlobDescriptor descriptor = new BlobDescriptor("/test/path", 10L, 20L); - BlobReference reference = new BlobReference(descriptor, "default.source", 7, 5L); + BlobReference reference = new BlobReference("default.source", 7, 5L); byte[] bytes = reference.serialize(); - bytes[0] = 2; + bytes[0] = 3; assertThatThrownBy(() -> BlobReference.deserialize(bytes)) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("Expecting BlobReference version to be 1"); } + + @Test + public void testEquality() { + BlobReference a = new BlobReference("default.source", 7, 5L); + BlobReference b = new BlobReference("default.source", 7, 5L); + BlobReference c = new BlobReference("default.source", 8, 5L); + + assertThat(a).isEqualTo(b); + assertThat(a.hashCode()).isEqualTo(b.hashCode()); + assertThat(a).isNotEqualTo(c); + } + + @Test + public void testIsBlobReference() { + BlobReference reference = new BlobReference("default.source", 7, 5L); + byte[] bytes = reference.serialize(); + + assertThat(BlobReference.isBlobReference(bytes)).isTrue(); + assertThat(BlobReference.isBlobReference(null)).isFalse(); + assertThat(BlobReference.isBlobReference(new byte[] {1, 2, 3})).isFalse(); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/BlobReferenceLookupFile.java b/paimon-core/src/main/java/org/apache/paimon/io/BlobReferenceLookupFile.java new file mode 100644 index 000000000000..2819f2a75d42 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/BlobReferenceLookupFile.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +/** Extra file storing deduplicated blob reference keys for a data file. */ +public class BlobReferenceLookupFile { + + public static final String PATH_SUFFIX = ".blobref"; + + private static final long MAGIC = 0x424C4F4252455831L; // "BLOBREX1" + private static final int VERSION = 1; + + public static Path path(Path dataFilePath) { + return new Path(dataFilePath.getParent(), dataFilePath.getName() + PATH_SUFFIX); + } + + public static boolean isLookupFile(String fileName) { + return fileName.endsWith(PATH_SUFFIX); + } + + public static void write(FileIO fileIO, Path path, Collection references) + throws IOException { + LinkedHashSet distinct = new LinkedHashSet<>(references); + + PositionOutputStream output = fileIO.newOutputStream(path, false); + try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(output)) { + out.writeLong(MAGIC); + out.writeInt(VERSION); + out.writeInt(distinct.size()); + for (BlobReference reference : distinct) { + out.writeUTF(reference.tableName()); + out.writeInt(reference.fieldId()); + out.writeLong(reference.rowId()); + } + } + } + + public static List read(FileIO fileIO, Path path) throws IOException { + SeekableInputStream input = fileIO.newInputStream(path); + try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(input)) { + long magic = in.readLong(); + if (magic != MAGIC) { + throw new IllegalArgumentException( + "Invalid blob reference lookup file: " + path + "."); + } + + int version = in.readInt(); + if (version != VERSION) { + throw new UnsupportedOperationException( + "Unsupported blob reference lookup file version: " + version + "."); + } + + int size = in.readInt(); + List references = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + references.add(new BlobReference(in.readUTF(), in.readInt(), in.readLong())); + } + return references; + } + } + + private BlobReferenceLookupFile() {} +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java index 7f8715ab0846..877e03bc3c0f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java @@ -18,6 +18,9 @@ package org.apache.paimon.io; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.BlobReferenceBlob; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.fs.FileIO; @@ -25,6 +28,7 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.stats.SimpleStatsConverter; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.LongCounter; import org.apache.paimon.utils.Pair; @@ -32,7 +36,8 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.util.Collections; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.function.Function; import java.util.function.Supplier; @@ -52,6 +57,9 @@ public class RowDataFileWriter extends StatsCollectingSingleFileWriter writeCols; + private final int[] blobRefFields; + private final LinkedHashSet blobReferences; + private @Nullable String blobReferenceLookupFileName; public RowDataFileWriter( FileIO fileIO, @@ -76,11 +84,18 @@ public RowDataFileWriter( fileIO, dataFileToFileIndexPath(path), writeSchema, fileIndexOptions); this.fileSource = fileSource; this.writeCols = writeCols; + this.blobRefFields = + writeSchema.getFields().stream() + .filter(field -> field.type().is(DataTypeRoot.BLOB_REF)) + .mapToInt(field -> writeSchema.getFieldIndex(field.name())) + .toArray(); + this.blobReferences = new LinkedHashSet<>(); } @Override public void write(InternalRow row) throws IOException { super.write(row); + collectBlobReferences(row); // add row to index if needed if (dataFileIndexWriter != null) { dataFileIndexWriter.write(row); @@ -106,6 +121,14 @@ public DataFileMeta result() throws IOException { ? DataFileIndexWriter.EMPTY_RESULT : dataFileIndexWriter.result(); String externalPath = isExternalPath ? path.toString() : null; + List extraFiles = new ArrayList<>(); + if (indexResult.independentIndexFile() != null) { + extraFiles.add(indexResult.independentIndexFile()); + } + String blobReferenceFile = writeBlobReferenceLookupFile(); + if (blobReferenceFile != null) { + extraFiles.add(blobReferenceFile); + } return DataFileMeta.forAppend( path.getName(), fileSize, @@ -114,9 +137,7 @@ public DataFileMeta result() throws IOException { seqNumCounter.getValue() - super.recordCount(), seqNumCounter.getValue() - 1, schemaId, - indexResult.independentIndexFile() == null - ? Collections.emptyList() - : Collections.singletonList(indexResult.independentIndexFile()), + extraFiles, indexResult.embeddedIndexBytes(), fileSource, statsPair.getKey(), @@ -124,4 +145,37 @@ public DataFileMeta result() throws IOException { null, writeCols); } + + private void collectBlobReferences(InternalRow row) { + for (int field : blobRefFields) { + Blob blob = row.getBlob(field); + if (blob == null) { + continue; + } + + if (!(blob instanceof BlobReferenceBlob)) { + throw new IllegalArgumentException( + "BLOB_REF fields only accept BlobReferenceBlob values, but found " + + blob.getClass().getSimpleName() + + "."); + } + + blobReferences.add(((BlobReferenceBlob) blob).reference()); + } + } + + @Nullable + private String writeBlobReferenceLookupFile() throws IOException { + if (blobReferences.isEmpty()) { + return null; + } + if (blobReferenceLookupFileName != null) { + return blobReferenceLookupFileName; + } + + Path lookupFilePath = BlobReferenceLookupFile.path(path); + BlobReferenceLookupFile.write(fileIO, lookupFilePath, blobReferences); + blobReferenceLookupFileName = lookupFilePath.getName(); + return blobReferenceLookupFileName; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java index 8cb70824cd45..77fb8aaf0ed0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java @@ -124,7 +124,7 @@ public InnerTableRead newRead() { () -> store().newRead(), config)); } return new AppendTableRead( - providerFactories, schema(), catalogEnvironment().catalogContext()); + providerFactories, schema(), catalogEnvironment().catalogContext(), fileIO()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java index 411bcf6767de..3dbf09b7753a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java @@ -150,7 +150,8 @@ public InnerTableRead newRead() { () -> store().newRead(), () -> store().newBatchRawFileRead(), schema(), - catalogEnvironment().catalogContext()); + catalogEnvironment().catalogContext(), + fileIO()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 0ed961487f80..020b670a3a2b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BlobReferenceInternalRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; import org.apache.paimon.reader.RecordReader; @@ -32,7 +33,8 @@ import org.apache.paimon.utils.BlobReferenceLookup; import org.apache.paimon.utils.ListUtils; import org.apache.paimon.utils.ProjectedRow; -import org.apache.paimon.utils.UriReaderFactory; + +import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; @@ -51,10 +53,13 @@ public abstract class AbstractDataTableRead implements InnerTableRead { private Predicate predicate; private final TableSchema schema; private final CatalogContext catalogContext; + @Nullable private final FileIO fileIO; - public AbstractDataTableRead(TableSchema schema, CatalogContext catalogContext) { + public AbstractDataTableRead( + TableSchema schema, CatalogContext catalogContext, @Nullable FileIO fileIO) { this.schema = schema; this.catalogContext = catalogContext; + this.fileIO = fileIO; } public abstract void applyReadType(RowType readType); @@ -114,6 +119,7 @@ public final RecordReader createReader(Split split) throws IOExcept } if (catalogContext != null) { + final Split finalSplit = split; RowType rowType = this.readType == null ? schema.logicalRowType() : this.readType; int[] blobRefFields = rowType.getFields().stream() @@ -121,16 +127,14 @@ public final RecordReader createReader(Split split) throws IOExcept .mapToInt(field -> rowType.getFieldIndex(field.name())) .toArray(); if (blobRefFields.length > 0) { - UriReaderFactory uriReaderFactory = new UriReaderFactory(catalogContext); reader = reader.transform( row -> new BlobReferenceInternalRow( row, blobRefFields, - uriReaderFactory, BlobReferenceLookup.createResolver( - catalogContext))); + catalogContext, fileIO, finalSplit))); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java index 388ef9344c1e..603452b1ff07 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendTableRead.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.operation.MergeFileSplitRead; import org.apache.paimon.operation.SplitRead; import org.apache.paimon.predicate.Predicate; @@ -53,8 +54,9 @@ public final class AppendTableRead extends AbstractDataTableRead { public AppendTableRead( List> providerFactories, TableSchema schema, - CatalogContext catalogContext) { - super(schema, catalogContext); + CatalogContext catalogContext, + FileIO fileIO) { + super(schema, catalogContext, fileIO); this.readProviders = providerFactories.stream() .map(factory -> factory.apply(this::config)) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java index ac83737afdc8..64dc87e8594a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java @@ -24,6 +24,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.operation.MergeFileSplitRead; import org.apache.paimon.operation.RawFileSplitRead; import org.apache.paimon.operation.SplitRead; @@ -65,8 +66,9 @@ public KeyValueTableRead( Supplier mergeReadSupplier, Supplier batchRawReadSupplier, TableSchema schema, - CatalogContext catalogContext) { - super(schema, catalogContext); + CatalogContext catalogContext, + FileIO fileIO) { + super(schema, catalogContext, fileIO); this.readProviders = Arrays.asList( new PrimaryKeyTableRawFileSplitReadProvider( diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java index 765c71970d05..ea27f6e764bb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java @@ -26,15 +26,56 @@ import org.apache.paimon.data.BlobReference; import org.apache.paimon.data.BlobReferenceResolver; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.io.BlobReferenceLookupFile; +import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; /** Utilities for resolving {@link BlobReference} through table metadata. */ public class BlobReferenceLookup { + private static final Logger LOG = LoggerFactory.getLogger(BlobReferenceLookup.class); + + /** + * Creates a resolver that uses the preloaded blob map from the split's extra files. If the + * preload map does not contain the requested reference, falls back to scanning the upstream + * table individually. + */ + public static BlobReferenceResolver createResolver( + CatalogContext catalogContext, @Nullable FileIO fileIO, Split split) { + Map cached = preload(catalogContext, fileIO, split); + if (cached.isEmpty()) { + return createResolver(catalogContext); + } + return reference -> { + Blob blob = cached.get(reference); + return blob != null ? blob : resolve(catalogContext, reference); + }; + } + + /** Creates a simple resolver that scans the upstream table for each reference individually. */ public static BlobReferenceResolver createResolver(CatalogContext catalogContext) { return reference -> resolve(catalogContext, reference); } @@ -83,7 +124,158 @@ public static Blob resolve(CatalogContext catalogContext, BlobReference referenc + reference.fieldId() + "."); } catch (Exception e) { - throw new RuntimeException("Failed to resolve blob reference fallback.", e); + throw new RuntimeException("Failed to resolve blob reference.", e); + } + } + + private static Map preload( + CatalogContext catalogContext, @Nullable FileIO fileIO, Split split) { + if (fileIO == null || !(split instanceof DataSplit)) { + return Collections.emptyMap(); + } + + List references = readLookupFiles(fileIO, (DataSplit) split); + if (references.isEmpty()) { + return Collections.emptyMap(); + } + + try (Catalog catalog = CatalogFactory.createCatalog(catalogContext)) { + return loadReferencedBlobs(catalog, references); + } catch (Exception e) { + LOG.warn("Failed to preload blob references from split extra files. Falling back.", e); + return Collections.emptyMap(); + } + } + + private static List readLookupFiles(FileIO fileIO, DataSplit split) { + LinkedHashSet references = new LinkedHashSet<>(); + Path defaultParent = new Path(split.bucketPath()); + for (DataFileMeta file : split.dataFiles()) { + Path parent = file.externalPathDir().map(Path::new).orElse(defaultParent); + for (String extraFile : file.extraFiles()) { + if (!BlobReferenceLookupFile.isLookupFile(extraFile)) { + continue; + } + + Path lookupFilePath = new Path(parent, extraFile); + try { + references.addAll(BlobReferenceLookupFile.read(fileIO, lookupFilePath)); + } catch (Exception e) { + LOG.warn( + "Failed to read blob reference lookup file {}. Skipping preload for this file.", + lookupFilePath, + e); + } + } + } + return new ArrayList<>(references); + } + + private static Map loadReferencedBlobs( + Catalog catalog, Collection references) throws Exception { + Map grouped = new HashMap<>(); + for (BlobReference reference : references) { + grouped.computeIfAbsent(reference.tableName(), TableReferences::new).add(reference); + } + + Map resolved = new HashMap<>(); + for (TableReferences tableReferences : grouped.values()) { + loadTableReferences(catalog, tableReferences, resolved); + } + return resolved; + } + + private static void loadTableReferences( + Catalog catalog, TableReferences tableReferences, Map resolved) + throws Exception { + Table table = catalog.getTable(Identifier.fromString(tableReferences.tableName)); + + List fields = new ArrayList<>(tableReferences.referencesByField.size()); + TreeSet rowIds = new TreeSet<>(); + for (Map.Entry> entry : + tableReferences.referencesByField.entrySet()) { + int fieldId = entry.getKey(); + if (!table.rowType().containsField(fieldId)) { + throw new IllegalArgumentException( + "Cannot find blob fieldId " + + fieldId + + " in upstream table " + + tableReferences.tableName + + "."); + } + + int fieldPos = table.rowType().getFieldIndexByFieldId(fieldId); + fields.add(new FieldRead(fieldId, fieldPos, table.rowType().getField(fieldPos))); + for (BlobReference reference : entry.getValue()) { + rowIds.add(reference.rowId()); + } + } + + Collections.sort(fields, (left, right) -> Integer.compare(left.fieldPos, right.fieldPos)); + + List readFields = new ArrayList<>(fields.size()); + for (FieldRead field : fields) { + readFields.add(field.field); + } + + ReadBuilder readBuilder = + table.newReadBuilder() + .withReadType(SpecialFields.rowTypeWithRowId(new RowType(readFields))) + .withRowRanges(Range.toRanges(rowIds)); + + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + RecordReader.RecordIterator batch; + while ((batch = reader.readBatch()) != null) { + try { + InternalRow row; + while ((row = batch.next()) != null) { + long rowId = row.getLong(fields.size()); + for (int i = 0; i < fields.size(); i++) { + Blob blob = row.getBlob(i); + if (blob != null) { + resolved.put( + new BlobReference( + tableReferences.tableName, + fields.get(i).fieldId, + rowId), + blob); + } + } + } + } finally { + batch.releaseBatch(); + } + } + } + } + + private static class TableReferences { + + private final String tableName; + private final Map> referencesByField = new HashMap<>(); + + private TableReferences(String tableName) { + this.tableName = tableName; + } + + private void add(BlobReference reference) { + referencesByField + .computeIfAbsent(reference.fieldId(), unused -> new ArrayList<>()) + .add(reference); + } + } + + private static class FieldRead { + + private final int fieldId; + private final int fieldPos; + private final DataField field; + + private FieldRead(int fieldId, int fieldPos, DataField field) { + this.fieldId = fieldId; + this.fieldPos = fieldPos; + this.field = field; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/io/RowDataFileWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/RowDataFileWriterTest.java new file mode 100644 index 000000000000..db9be7507479 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/io/RowDataFileWriterTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fileindex.FileIndexOptions; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.options.Options; +import org.apache.paimon.statistics.NoneSimpleColStatsCollector; +import org.apache.paimon.statistics.SimpleColStatsCollector; +import org.apache.paimon.types.BlobRefType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.LongCounter; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RowDataFileWriter}. */ +public class RowDataFileWriterTest { + + private static final RowType SCHEMA = + RowType.of( + new DataType[] {new IntType(), new BlobRefType()}, + new String[] {"id", "ref"}); + + @TempDir java.nio.file.Path tempDir; + + @Test + public void testWriteBlobReferenceLookupFile() throws Exception { + FileFormat fileFormat = FileFormat.fromIdentifier("parquet", new Options()); + Path dataPath = new Path(tempDir.toUri().toString(), "data.parquet"); + BlobReference reference = new BlobReference("default.upstream", 7, 11L); + + RowDataFileWriter writer = + new RowDataFileWriter( + LocalFileIO.create(), + RollingFileWriter.createFileWriterContext( + fileFormat, + SCHEMA, + new SimpleColStatsCollector.Factory[] { + NoneSimpleColStatsCollector::new, + NoneSimpleColStatsCollector::new + }, + CoreOptions.FILE_COMPRESSION.defaultValue()), + dataPath, + SCHEMA, + 0L, + () -> new LongCounter(0), + new FileIndexOptions(), + FileSource.APPEND, + false, + false, + false, + SCHEMA.getFieldNames()); + + writer.write(GenericRow.of(1, Blob.fromReference(null, reference))); + writer.close(); + + DataFileMeta meta = writer.result(); + Path lookupPath = BlobReferenceLookupFile.path(dataPath); + + assertThat(meta.extraFiles()).contains(lookupPath.getName()); + assertThat(LocalFileIO.create().exists(lookupPath)).isTrue(); + assertThat(BlobReferenceLookupFile.read(LocalFileIO.create(), lookupPath)) + .isEqualTo(Collections.singletonList(reference)); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java index 3a6dccb00854..3e855d76863b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java @@ -21,15 +21,21 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.Blob; -import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.BlobReference; +import org.apache.paimon.data.BlobReferenceResolver; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.io.BlobReferenceLookupFile; +import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.metrics.MetricRegistry; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableRead; @@ -39,6 +45,7 @@ import org.apache.paimon.types.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -46,17 +53,22 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.OptionalLong; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** Tests for {@link BlobReferenceLookup}. */ public class BlobReferenceLookupTest { + @TempDir java.nio.file.Path tempDir; + @Test public void testResolveByFieldIdAfterRename() throws Exception { CatalogContext context = mock(CatalogContext.class); @@ -79,14 +91,12 @@ public void testResolveByFieldIdAfterRename() throws Exception { when(table.newReadBuilder()).thenReturn(readBuilder); when(readBuilder.withProjection(any(int[].class))).thenReturn(readBuilder); when(readBuilder.withRowRanges(anyList())).thenReturn(readBuilder); - when(readBuilder.newRead()).thenReturn(new SingleRowTableRead(split, row)); + when(readBuilder.newRead()).thenReturn(new ListRowTableRead(split, Collections.singletonList(row))); when(readBuilder.newScan()).thenReturn(scan); when(scan.plan()).thenReturn(plan); when(plan.splits()).thenReturn(Collections.singletonList(split)); - BlobReference reference = - new BlobReference( - new BlobDescriptor("file:/missing", 0L, -1L), "default.source", 7, 12L); + BlobReference reference = new BlobReference("default.source", 7, 12L); try (MockedStatic mockedCatalogFactory = Mockito.mockStatic(CatalogFactory.class)) { @@ -100,14 +110,90 @@ public void testResolveByFieldIdAfterRename() throws Exception { } } - private static class SingleRowTableRead implements TableRead { + @Test + public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { + CatalogContext context = mock(CatalogContext.class); + Catalog catalog = mock(Catalog.class); + Table table = mock(Table.class); + ReadBuilder readBuilder = mock(ReadBuilder.class); + TableScan scan = mock(TableScan.class); + TableScan.Plan plan = mock(TableScan.Plan.class); + + byte[] leftPayload = new byte[] {1, 2, 3}; + byte[] rightPayload = new byte[] {4, 5, 6}; + BlobReference leftReference = new BlobReference("default.source", 7, 12L); + BlobReference rightReference = new BlobReference("default.source", 8, 12L); + + Path bucketPath = new Path(tempDir.toUri()); + Path lookupFilePath = new Path(bucketPath, "data-0.parquet" + BlobReferenceLookupFile.PATH_SUFFIX); + BlobReferenceLookupFile.write( + LocalFileIO.create(), + lookupFilePath, + java.util.Arrays.asList(leftReference, rightReference)); + + DataFileMeta dataFile = mock(DataFileMeta.class); + when(dataFile.extraFiles()).thenReturn(Collections.singletonList(lookupFilePath.getName())); + when(dataFile.externalPathDir()).thenReturn(java.util.Optional.empty()); + + DataSplit split = + DataSplit.builder() + .withSnapshot(1L) + .withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0) + .withBucketPath(bucketPath.toString()) + .withDataFiles(Collections.singletonList(dataFile)) + .isStreaming(false) + .rawConvertible(false) + .build(); + + Split readerSplit = new TestSplit(); + InternalRow preloadRow = + GenericRow.of(Blob.fromData(leftPayload), Blob.fromData(rightPayload), 12L); + + when(catalog.getTable(any())).thenReturn(table); + when(table.rowType()) + .thenReturn( + new RowType( + java.util.Arrays.asList( + new DataField(7, "blob_left", DataTypes.BLOB()), + new DataField(8, "blob_right", DataTypes.BLOB())))); + when(table.newReadBuilder()).thenReturn(readBuilder); + when(readBuilder.withReadType(any(RowType.class))).thenReturn(readBuilder); + when(readBuilder.withRowRanges(anyList())).thenReturn(readBuilder); + when(readBuilder.newRead()) + .thenReturn(new ListRowTableRead(readerSplit, Collections.singletonList(preloadRow))); + when(readBuilder.newScan()).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.splits()).thenReturn(Collections.singletonList(readerSplit)); + + try (MockedStatic mockedCatalogFactory = + Mockito.mockStatic(CatalogFactory.class)) { + mockedCatalogFactory + .when(() -> CatalogFactory.createCatalog(context)) + .thenReturn(catalog); + + BlobReferenceResolver resolver = + BlobReferenceLookup.createResolver(context, LocalFileIO.create(), split); + + assertThat(resolver.resolve(leftReference).toData()).isEqualTo(leftPayload); + assertThat(resolver.resolve(rightReference).toData()).isEqualTo(rightPayload); + assertThat(resolver.resolve(new BlobReference("default.source", 7, 12L)).toData()) + .isEqualTo(leftPayload); + assertThat(resolver.resolve(new BlobReference("default.source", 8, 12L)).toData()) + .isEqualTo(rightPayload); + + verify(table, times(1)).newReadBuilder(); + } + } + + private static class ListRowTableRead implements TableRead { private final Split split; - private final InternalRow row; + private final List rows; - private SingleRowTableRead(Split split, InternalRow row) { + private ListRowTableRead(Split split, List rows) { this.split = split; - this.row = row; + this.rows = rows; } @Override @@ -141,16 +227,12 @@ public RecordIterator readBatch() { emitted = true; return new RecordIterator() { - private boolean rowReturned = false; + private int next = 0; @Nullable @Override public InternalRow next() { - if (rowReturned) { - return null; - } - rowReturned = true; - return row; + return next < rows.size() ? rows.get(next++) : null; } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java index 76d83393b3d4..49653a0ce35b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java @@ -44,7 +44,7 @@ public class LookupCompactDiffRead extends AbstractDataTableRead { public LookupCompactDiffRead( MergeFileSplitRead mergeRead, TableSchema schema, CatalogContext catalogContext) { - super(schema, catalogContext); + super(schema, catalogContext, null); this.incrementalDiffRead = new IncrementalCompactDiffSplitRead(mergeRead); this.fullPhaseMergeRead = SplitRead.convert( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java index a0d5f98ba0e0..f110c7e31488 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java @@ -152,7 +152,7 @@ public KeyValueTableRead createReadWithKey() { FileFormatDiscover.of(options), pathFactory, options); - return new KeyValueTableRead(() -> read, () -> rawFileRead, schema, null); + return new KeyValueTableRead(() -> read, () -> rawFileRead, schema, null, null); } public List writeFiles( diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 10e403eb154a..0e8ffc291c05 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -19,7 +19,6 @@ package org.apache.paimon.format.avro; import org.apache.paimon.data.Blob; -import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.BlobReference; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -220,11 +219,10 @@ void testBlobRefRoundTrip() throws IOException { RowType rowType = DataTypes.ROW(DataTypes.FIELD(0, "blob_ref", DataTypes.BLOB_REF())); BlobReference reference = new BlobReference( - new BlobDescriptor("file:/tmp/blob-ref", 12L, 34L), "default.t", 7, 11L); - Blob blob = Blob.fromReference(null, null, reference); + Blob blob = Blob.fromReference(null, reference); FileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); LocalFileIO fileIO = LocalFileIO.create(); From 5bc083b9302355468bc6a74819dc592a3008f819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Fri, 10 Apr 2026 16:26:22 +0800 Subject: [PATCH 3/3] Kiro 02 --- .../paimon/utils/BlobReferenceLookup.java | 58 +++++++++++------- .../paimon/utils/BlobReferenceLookupTest.java | 61 +++++++++++++++---- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java index ea27f6e764bb..6a5576c2802c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/BlobReferenceLookup.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.BlobReference; import org.apache.paimon.data.BlobReferenceResolver; import org.apache.paimon.data.InternalRow; @@ -52,26 +53,38 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; - -/** Utilities for resolving {@link BlobReference} through table metadata. */ +/** + * Utilities for resolving {@link BlobReference} through table metadata. + * + *

The preload phase only caches lightweight {@link BlobDescriptor} (uri + offset + length) + * rather than the actual blob data, so memory usage stays small even when a data file contains a + * large number of blob references. + */ public class BlobReferenceLookup { private static final Logger LOG = LoggerFactory.getLogger(BlobReferenceLookup.class); /** - * Creates a resolver that uses the preloaded blob map from the split's extra files. If the - * preload map does not contain the requested reference, falls back to scanning the upstream - * table individually. + * Creates a resolver that first checks the preloaded descriptor cache built from the split's + * extra {@code .blobref} files. On a cache hit the descriptor is used to construct a {@link + * Blob} directly (no scan needed). On a cache miss the resolver falls back to scanning the + * upstream table for the individual reference. */ public static BlobReferenceResolver createResolver( CatalogContext catalogContext, @Nullable FileIO fileIO, Split split) { - Map cached = preload(catalogContext, fileIO, split); + Map cached = + preloadDescriptors(catalogContext, fileIO, split); + UriReaderFactory uriReaderFactory = new UriReaderFactory(catalogContext); if (cached.isEmpty()) { return createResolver(catalogContext); } return reference -> { - Blob blob = cached.get(reference); - return blob != null ? blob : resolve(catalogContext, reference); + BlobDescriptor descriptor = cached.get(reference); + if (descriptor != null) { + return Blob.fromDescriptor( + uriReaderFactory.create(descriptor.uri()), descriptor); + } + return resolve(catalogContext, reference); }; } @@ -128,7 +141,7 @@ public static Blob resolve(CatalogContext catalogContext, BlobReference referenc } } - private static Map preload( + private static Map preloadDescriptors( CatalogContext catalogContext, @Nullable FileIO fileIO, Split split) { if (fileIO == null || !(split instanceof DataSplit)) { return Collections.emptyMap(); @@ -140,9 +153,10 @@ private static Map preload( } try (Catalog catalog = CatalogFactory.createCatalog(catalogContext)) { - return loadReferencedBlobs(catalog, references); + return loadReferencedDescriptors(catalog, references); } catch (Exception e) { - LOG.warn("Failed to preload blob references from split extra files. Falling back.", e); + LOG.warn( + "Failed to preload blob descriptors from split extra files. Falling back.", e); return Collections.emptyMap(); } } @@ -156,13 +170,13 @@ private static List readLookupFiles(FileIO fileIO, DataSplit spli if (!BlobReferenceLookupFile.isLookupFile(extraFile)) { continue; } - Path lookupFilePath = new Path(parent, extraFile); try { references.addAll(BlobReferenceLookupFile.read(fileIO, lookupFilePath)); } catch (Exception e) { LOG.warn( - "Failed to read blob reference lookup file {}. Skipping preload for this file.", + "Failed to read blob reference lookup file {}." + + " Skipping preload for this file.", lookupFilePath, e); } @@ -171,22 +185,23 @@ private static List readLookupFiles(FileIO fileIO, DataSplit spli return new ArrayList<>(references); } - private static Map loadReferencedBlobs( + private static Map loadReferencedDescriptors( Catalog catalog, Collection references) throws Exception { Map grouped = new HashMap<>(); for (BlobReference reference : references) { grouped.computeIfAbsent(reference.tableName(), TableReferences::new).add(reference); } - - Map resolved = new HashMap<>(); + Map resolved = new HashMap<>(); for (TableReferences tableReferences : grouped.values()) { - loadTableReferences(catalog, tableReferences, resolved); + loadTableDescriptors(catalog, tableReferences, resolved); } return resolved; } - private static void loadTableReferences( - Catalog catalog, TableReferences tableReferences, Map resolved) + private static void loadTableDescriptors( + Catalog catalog, + TableReferences tableReferences, + Map resolved) throws Exception { Table table = catalog.getTable(Identifier.fromString(tableReferences.tableName)); @@ -203,7 +218,6 @@ private static void loadTableReferences( + tableReferences.tableName + "."); } - int fieldPos = table.rowType().getFieldIndexByFieldId(fieldId); fields.add(new FieldRead(fieldId, fieldPos, table.rowType().getField(fieldPos))); for (BlobReference reference : entry.getValue()) { @@ -239,7 +253,7 @@ private static void loadTableReferences( tableReferences.tableName, fields.get(i).fieldId, rowId), - blob); + blob.toDescriptor()); } } } @@ -251,7 +265,6 @@ private static void loadTableReferences( } private static class TableReferences { - private final String tableName; private final Map> referencesByField = new HashMap<>(); @@ -267,7 +280,6 @@ private void add(BlobReference reference) { } private static class FieldRead { - private final int fieldId; private final int fieldPos; private final DataField field; diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java index 3e855d76863b..ed11cbadd89b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/BlobReferenceLookupTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.BlobReference; import org.apache.paimon.data.BlobReferenceResolver; import org.apache.paimon.data.GenericRow; @@ -91,7 +92,9 @@ public void testResolveByFieldIdAfterRename() throws Exception { when(table.newReadBuilder()).thenReturn(readBuilder); when(readBuilder.withProjection(any(int[].class))).thenReturn(readBuilder); when(readBuilder.withRowRanges(anyList())).thenReturn(readBuilder); - when(readBuilder.newRead()).thenReturn(new ListRowTableRead(split, Collections.singletonList(row))); + when(readBuilder.newRead()) + .thenReturn( + new ListRowTableRead(split, Collections.singletonList(row))); when(readBuilder.newScan()).thenReturn(scan); when(scan.plan()).thenReturn(plan); when(plan.splits()).thenReturn(Collections.singletonList(split)); @@ -105,13 +108,12 @@ public void testResolveByFieldIdAfterRename() throws Exception { .thenReturn(catalog); Blob resolved = BlobReferenceLookup.resolve(context, reference); - assertThat(resolved.toData()).isEqualTo(payload); } } @Test - public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { + public void testCreateResolverPreloadsDescriptorsFromSplit() throws Exception { CatalogContext context = mock(CatalogContext.class); Catalog catalog = mock(Catalog.class); Table table = mock(Table.class); @@ -119,20 +121,48 @@ public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { TableScan scan = mock(TableScan.class); TableScan.Plan plan = mock(TableScan.Plan.class); + // Write real blob files so that the preloaded descriptors point to readable data. byte[] leftPayload = new byte[] {1, 2, 3}; byte[] rightPayload = new byte[] {4, 5, 6}; + Path leftBlobPath = new Path(tempDir.toUri().toString(), "left.blob"); + Path rightBlobPath = new Path(tempDir.toUri().toString(), "right.blob"); + LocalFileIO fileIO = LocalFileIO.create(); + try (org.apache.paimon.fs.PositionOutputStream out = + fileIO.newOutputStream(leftBlobPath, false)) { + out.write(leftPayload); + } + try (org.apache.paimon.fs.PositionOutputStream out = + fileIO.newOutputStream(rightBlobPath, false)) { + out.write(rightPayload); + } + + BlobDescriptor leftDescriptor = + new BlobDescriptor(leftBlobPath.toString(), 0L, leftPayload.length); + BlobDescriptor rightDescriptor = + new BlobDescriptor(rightBlobPath.toString(), 0L, rightPayload.length); + + // The upstream table returns BlobRef (descriptor-backed) blobs, not inline BlobData. + Blob leftBlob = + Blob.fromDescriptor(UriReader.fromFile(LocalFileIO.create()), leftDescriptor); + Blob rightBlob = + Blob.fromDescriptor(UriReader.fromFile(LocalFileIO.create()), rightDescriptor); + BlobReference leftReference = new BlobReference("default.source", 7, 12L); BlobReference rightReference = new BlobReference("default.source", 8, 12L); Path bucketPath = new Path(tempDir.toUri()); - Path lookupFilePath = new Path(bucketPath, "data-0.parquet" + BlobReferenceLookupFile.PATH_SUFFIX); + Path lookupFilePath = + new Path( + bucketPath, + "data-0.parquet" + BlobReferenceLookupFile.PATH_SUFFIX); BlobReferenceLookupFile.write( LocalFileIO.create(), lookupFilePath, java.util.Arrays.asList(leftReference, rightReference)); DataFileMeta dataFile = mock(DataFileMeta.class); - when(dataFile.extraFiles()).thenReturn(Collections.singletonList(lookupFilePath.getName())); + when(dataFile.extraFiles()) + .thenReturn(Collections.singletonList(lookupFilePath.getName())); when(dataFile.externalPathDir()).thenReturn(java.util.Optional.empty()); DataSplit split = @@ -147,8 +177,7 @@ public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { .build(); Split readerSplit = new TestSplit(); - InternalRow preloadRow = - GenericRow.of(Blob.fromData(leftPayload), Blob.fromData(rightPayload), 12L); + InternalRow preloadRow = GenericRow.of(leftBlob, rightBlob, 12L); when(catalog.getTable(any())).thenReturn(table); when(table.rowType()) @@ -161,7 +190,9 @@ public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { when(readBuilder.withReadType(any(RowType.class))).thenReturn(readBuilder); when(readBuilder.withRowRanges(anyList())).thenReturn(readBuilder); when(readBuilder.newRead()) - .thenReturn(new ListRowTableRead(readerSplit, Collections.singletonList(preloadRow))); + .thenReturn( + new ListRowTableRead( + readerSplit, Collections.singletonList(preloadRow))); when(readBuilder.newScan()).thenReturn(scan); when(scan.plan()).thenReturn(plan); when(plan.splits()).thenReturn(Collections.singletonList(readerSplit)); @@ -173,15 +204,21 @@ public void testCreateResolverPreloadsReferencesFromSplit() throws Exception { .thenReturn(catalog); BlobReferenceResolver resolver = - BlobReferenceLookup.createResolver(context, LocalFileIO.create(), split); + BlobReferenceLookup.createResolver( + context, LocalFileIO.create(), split); + // The resolver should return blobs whose data matches the original payloads. assertThat(resolver.resolve(leftReference).toData()).isEqualTo(leftPayload); assertThat(resolver.resolve(rightReference).toData()).isEqualTo(rightPayload); - assertThat(resolver.resolve(new BlobReference("default.source", 7, 12L)).toData()) + + // Same coordinates → same result (cache hit). + assertThat( + resolver.resolve( + new BlobReference("default.source", 7, 12L)) + .toData()) .isEqualTo(leftPayload); - assertThat(resolver.resolve(new BlobReference("default.source", 8, 12L)).toData()) - .isEqualTo(rightPayload); + // Only one readBuilder should have been created (batch preload). verify(table, times(1)).newReadBuilder(); } }