diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp index 77ee39e80..bbe2f7b7a 100644 --- a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp @@ -45,16 +45,21 @@ Result BinaryToBlobCastExecutor::Cast( Result> BinaryToBlobCastExecutor::Cast( const std::shared_ptr& array, const std::shared_ptr& target_type, arrow::MemoryPool* pool) const { - if (array->type_id() != arrow::Type::BINARY) { - return Status::Invalid( - fmt::format("BinaryToBlobCastExecutor only supports binary input, got {}", - array->type()->ToString())); - } if (target_type->id() != arrow::Type::LARGE_BINARY) { return Status::Invalid( fmt::format("BinaryToBlobCastExecutor only supports large_binary target, got {}", target_type->ToString())); } + // Parquet may have already restored the logical BLOB representation from ARROW:schema. + if (array->type_id() == arrow::Type::LARGE_BINARY) { + return array; + } + if (array->type_id() != arrow::Type::BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports binary or large_binary input, " + "got {}", + array->type()->ToString())); + } auto binary_array = checked_pointer_cast(array); if (binary_array->offset() != 0) { diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index 3905ba04d..9035eb295 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -1427,6 +1427,18 @@ TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArray) { ASSERT_EQ(target_array->data()->buffers[2], src_array->data()->buffers[2]); } +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorAlreadyLargeBinary) { + auto cast_executor = std::make_shared(); + auto src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::large_binary(), R"(["foo", null, "bar"])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr target_array, + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool())); + ASSERT_EQ(target_array, src_array); +} + TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArrayWithOffset) { auto cast_executor = std::make_shared(); auto src_array = diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 139b7fb20..3e73294ff 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -805,6 +805,52 @@ TEST_F(FieldMappingReaderTest, TestReadInlineBlobAsBinaryDataFile) { /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); } +TEST_F(FieldMappingReaderTest, TestReadInlineBlobAlreadyRestoredAsLargeBinary) { + // The data schema declares the inline BLOB's physical type as binary, while the mock reader + // simulates Parquet restoring the actual array to logical large_binary from ARROW:schema. + std::vector data_fields = { + DataField(0, arrow::field("descriptor", arrow::binary(), /*nullable=*/true)), + }; + std::shared_ptr data_schema = + DataField::ConvertDataFieldsToArrowSchema(data_fields); + + std::vector read_fields = { + DataField(0, BlobUtils::ToArrowField("descriptor", /*nullable=*/true)), + }; + std::shared_ptr read_schema = + DataField::ConvertDataFieldsToArrowSchema(read_fields); + std::shared_ptr restored_array = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("descriptor", arrow::large_binary())}), + R"([ + ["descriptor-1"], + [null], + ["descriptor-2"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, + /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr mapping, + mapping_builder->CreateFieldMapping(data_fields)); + std::unique_ptr mock = std::make_unique( + restored_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + FieldMappingReader::Create(read_schema->num_fields(), std::move(mock), + BinaryRow::EmptyRow(), std::move(mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, + GetArrowPool(pool_))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + + std::shared_ptr expected = + std::make_shared(arrow::ArrayVector({restored_array})); + ASSERT_TRUE(result->type()->Equals(expected->type())) + << result->type()->ToString() << " vs " << expected->type()->ToString(); + ASSERT_TRUE(result->Equals(expected)) << result->ToString() << " vs " << expected->ToString(); +} + TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { // Test all 4 combinations of rename × cast: // f0: no rename, no cast (utf8 → utf8, name unchanged) diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.cpp b/src/paimon/format/parquet/parquet_timestamp_converter.cpp index fc628e60f..01521762b 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.cpp +++ b/src/paimon/format/parquet/parquet_timestamp_converter.cpp @@ -24,12 +24,26 @@ #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" namespace paimon::parquet { +namespace { + +bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { + // ARROW:schema may restore an inline BLOB's physical binary column to its logical + // large_binary representation before this compatibility check runs. + return src_field->type()->id() == arrow::Type::LARGE_BINARY && + target_field->type()->id() == arrow::Type::BINARY && + BlobUtils::IsBlobMetadata(target_field->metadata()); +} + +} // namespace + Result> ParquetTimestampConverter::AdjustTimezone( const std::shared_ptr& src_data_type) { arrow::Type::type type = src_data_type->id(); @@ -91,9 +105,14 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( src_data_type->ToString(), target_data_type->ToString())); } for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { - PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedCastArrayForTimestamp( - src_struct_type->field(i)->type(), - target_struct_type->field(i)->type())); + const std::shared_ptr& src_field = src_struct_type->field(i); + const std::shared_ptr& target_field = target_struct_type->field(i); + if (IsAlreadyLogicalBlob(src_field, target_field)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + bool need_cast, + NeedCastArrayForTimestamp(src_field->type(), target_field->type())); if (need_cast) { return true; } diff --git a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp b/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp index 4d764e1ad..784eecd05 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp @@ -23,6 +23,7 @@ #include "arrow/api.h" #include "arrow/ipc/api.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/memory/memory_pool.h" @@ -90,6 +91,24 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { } } +TEST(ParquetTimestampConverterTest, TestAlreadyLogicalBlobCompatibility) { + std::shared_ptr restored_blob = arrow::field("blob", arrow::large_binary()); + std::shared_ptr expected_physical_blob = + BlobUtils::ToArrowField("blob")->WithType(arrow::binary()); + + ASSERT_OK_AND_ASSIGN(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + arrow::struct_({restored_blob}), + arrow::struct_({expected_physical_blob}))); + ASSERT_FALSE(need_cast); + + // Only a binary field carrying Paimon BLOB metadata is compatible with a large_binary field + // already restored from ARROW:schema. A plain binary field must still be rejected. + ASSERT_NOK_WITH_MSG(ParquetTimestampConverter::NeedCastArrayForTimestamp( + arrow::struct_({restored_blob}), + arrow::struct_({arrow::field("blob", arrow::binary())})), + "src type large_binary and target type binary mismatch"); +} + TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 75147ce60..271a52c62 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -78,6 +78,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(paimon_read_compat_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(scan_and_read_inte_test STATIC_LINK_LIBS paimon_shared diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp new file mode 100644 index 000000000..94d6c99fc --- /dev/null +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -0,0 +1,508 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/defs.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" + +namespace paimon::test { +namespace { + +const char kDatabaseName[] = "append_types_compatibility"; + +std::vector WriterPrefixes() { + return {"python", "rust", "java"}; +} + +Result> ReadTable( + const std::string& table_name, const std::vector& field_names, + const std::map& options = {}) { + std::string table_path = GetDataDir() + "/parquet/" + kDatabaseName + ".db/" + table_name; + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + return ReadResultCollector::CollectResult(std::move(batch_reader)); +} + +std::shared_ptr GetOnlyStructChunk( + const std::shared_ptr& result) { + if (!result || result->num_chunks() != 1) { + return nullptr; + } + return std::dynamic_pointer_cast(result->chunk(0)); +} + +void AssertArrayEqualsJson(const std::shared_ptr& actual, const std::string& context, + const std::string& expected_json) { + ASSERT_TRUE(actual) << context; + arrow::Result> expected = + arrow::ipc::internal::json::ArrayFromJSON(actual->type(), expected_json); + ASSERT_TRUE(expected.ok()) << context << ": " << expected.status().ToString(); + ASSERT_TRUE(actual->Equals(*expected)) << context << "\nexpected: " << (*expected)->ToString() + << "\nactual: " << actual->ToString(); +} + +void AssertFieldEqualsJson(const std::shared_ptr& rows, + const std::string& field_name, + const std::shared_ptr& expected_type, + const std::string& expected_json) { + std::shared_ptr actual = rows->GetFieldByName(field_name); + ASSERT_TRUE(actual) << field_name; + ASSERT_TRUE(actual->type()->Equals(expected_type)) + << field_name << " expected type: " << expected_type->ToString() + << " actual type: " << actual->type()->ToString(); + AssertArrayEqualsJson(actual, field_name, expected_json); +} + +Result VariantJsonAt(const std::shared_ptr& variants, + int64_t index) { + if (variants->IsNull(index)) { + return std::string("null"); + } + std::shared_ptr values = + std::dynamic_pointer_cast(variants->GetFieldByName("value")); + std::shared_ptr metadata = + std::dynamic_pointer_cast(variants->GetFieldByName("metadata")); + if (!values || !metadata) { + return Status::Invalid("Expected unshredded VARIANT struct"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(values->GetView(index), metadata->GetView(index), GetDefaultPool())); + return variant->ToJson(); +} + +void AssertVariantJsonAt(const std::shared_ptr& variants, int64_t index, + const std::string& expected_json) { + ASSERT_OK_AND_ASSIGN(std::string actual_json, VariantJsonAt(variants, index)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(expected_json, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(std::string normalized_expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, normalized_expected_json); +} + +} // namespace + +class PaimonReadCompatInteTest : public ::testing::TestWithParam {}; + +TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { + const std::string& writer_prefix = GetParam(); + TimezoneGuard timezone_guard("Asia/Shanghai"); + + // Project every non-BLOB field from the main compatibility table in schema order. + std::vector fields = { + // Basic numeric types. + "id", + "f_boolean", + "f_tinyint", + "f_smallint", + "f_int", + "f_bigint", + "f_float", + "f_double", + // Character and binary string types. + "f_char", + "f_varchar", + "f_string", + "f_binary", + "f_varbinary", + "f_bytes", + // DECIMAL values across physical precision boundaries. + "f_decimal_1_0", + "f_decimal_9_2", + "f_decimal_18_2", + "f_decimal_19_2", + "f_decimal_38_18", + "f_decimal_38_38", + // Date and timestamp types at every supported precision. + "f_date", + "f_timestamp_0", + "f_timestamp_3", + "f_timestamp_6", + "f_timestamp_9", + "f_timestamp_ltz_0", + "f_timestamp_ltz_3", + "f_timestamp_ltz_6", + "f_timestamp_ltz_9", + // VARIANT and nested container types. + "f_variant", + "f_array_int", + "f_map_string_bigint", + "f_row", + "f_array_array_int", + "f_array_map", + "f_map_array", + "f_array_row", + "f_map_row", + "f_deep_row", + "f_array_variant", + "f_map_variant", + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadTable(writer_prefix + "_types", fields)); + std::shared_ptr rows = GetOnlyStructChunk(result); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 3); + ASSERT_EQ(rows->num_fields(), static_cast(fields.size() + 1)); + + // Paimon row kind and basic numeric values. + AssertFieldEqualsJson(rows, "_VALUE_KIND", arrow::int8(), "[0, 0, 0]"); + AssertFieldEqualsJson(rows, "id", arrow::int32(), "[1, 2, 3]"); + AssertFieldEqualsJson(rows, "f_boolean", arrow::boolean(), "[true, null, null]"); + AssertFieldEqualsJson(rows, "f_tinyint", arrow::int8(), "[-8, null, null]"); + AssertFieldEqualsJson(rows, "f_smallint", arrow::int16(), "[1234, null, null]"); + AssertFieldEqualsJson(rows, "f_int", arrow::int32(), "[-123456, null, null]"); + AssertFieldEqualsJson(rows, "f_bigint", arrow::int64(), "[9223372036854770000, null, null]"); + AssertFieldEqualsJson(rows, "f_float", arrow::float32(), "[1.25, null, null]"); + AssertFieldEqualsJson(rows, "f_double", arrow::float64(), "[-12345.6789, null, null]"); + + // Character and binary strings, including empty and embedded-null values. + AssertFieldEqualsJson(rows, "f_char", arrow::utf8(), R"(["char", null, ""])"); + AssertFieldEqualsJson(rows, "f_varchar", arrow::utf8(), R"(["varchar-中文", null, ""])"); + AssertFieldEqualsJson(rows, "f_string", arrow::utf8(), R"(["pypaimon 2.0.0", null, ""])"); + AssertFieldEqualsJson(rows, "f_binary", arrow::binary(), R"(["12345678", null, "ABCDEFGH"])"); + AssertFieldEqualsJson(rows, "f_varbinary", arrow::binary(), + R"(["varbinary\u0000value", null, ""])"); + AssertFieldEqualsJson(rows, "f_bytes", arrow::binary(), R"(["bytes\u0000value", null, ""])"); + + // DECIMAL values around the int32, int64, and fixed-byte-array precision boundaries. + AssertFieldEqualsJson(rows, "f_decimal_1_0", arrow::decimal128(1, 0), R"(["9", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_9_2", arrow::decimal128(9, 2), + R"(["1234567.89", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_18_2", arrow::decimal128(18, 2), + R"(["1234567890123456.78", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_19_2", arrow::decimal128(19, 2), + R"(["12345678901234567.89", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_38_18", arrow::decimal128(38, 18), + R"(["12345678901234567890.123456789012345678", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_38_38", arrow::decimal128(38, 38), + R"(["0.12345678901234567890123456789012345678", null, null])"); + + // DATE, TIMESTAMP, and TIMESTAMP_LTZ values at second through nanosecond precision. + AssertFieldEqualsJson(rows, "f_date", arrow::date32(), "[19782, null, null]"); + AssertFieldEqualsJson(rows, "f_timestamp_0", arrow::timestamp(arrow::TimeUnit::SECOND), + R"(["2024-02-29 12:34:56", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_3", arrow::timestamp(arrow::TimeUnit::MILLI), + R"(["2024-02-29 12:34:56.123", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_6", arrow::timestamp(arrow::TimeUnit::MICRO), + R"(["2024-02-29 12:34:56.123456", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_9", arrow::timestamp(arrow::TimeUnit::NANO), + R"(["2024-02-29 12:34:56.123456000", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_0", + arrow::timestamp(arrow::TimeUnit::SECOND, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_3", + arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_6", + arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123456", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_9", + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123456000", null, null])"); + + // First-level ARRAY, MAP, and ROW values with null and empty containers. + AssertFieldEqualsJson(rows, "f_array_int", arrow::list(arrow::int32()), + "[[1, null, 3], null, []]"); + AssertFieldEqualsJson(rows, "f_map_string_bigint", arrow::map(arrow::utf8(), arrow::int64()), + R"([[ ["one", 1], ["null", null] ], null, []])"); + std::shared_ptr row_type = arrow::struct_({ + arrow::field("nested_int", arrow::int32()), + arrow::field("nested_string", arrow::utf8()), + arrow::field("nested_decimal", arrow::decimal128(19, 4)), + arrow::field("nested_timestamp", arrow::timestamp(arrow::TimeUnit::NANO)), + arrow::field("nested_timestamp_ltz", + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai")), + }); + AssertFieldEqualsJson( + rows, "f_row", row_type, + R"([[7, "nested", "123456789012345.6789", "2024-02-29 12:34:56.123456000", "2024-02-29 12:34:56.123456000"], null, null])"); + + // Nested ARRAY/MAP combinations and containers whose values are ROWs. + AssertFieldEqualsJson(rows, "f_array_array_int", arrow::list(arrow::list(arrow::int32())), + "[[[1, 2], null, []], null, []]"); + AssertFieldEqualsJson(rows, "f_array_map", + arrow::list(arrow::map(arrow::utf8(), arrow::int32())), + R"([[[["a", 1], ["b", null]], null, []], null, []])"); + AssertFieldEqualsJson(rows, "f_map_array", + arrow::map(arrow::utf8(), arrow::list(arrow::int32())), + R"([[ ["numbers", [1, null, 3]], ["empty", []] ], null, []])"); + std::shared_ptr array_row_type = arrow::list(arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::decimal128(9, 2)), + })); + AssertFieldEqualsJson(rows, "f_array_row", array_row_type, + R"([[ ["alice", "99.50"], null, ["bob", null] ], null, []])"); + std::shared_ptr map_row_type = arrow::map( + arrow::utf8(), + arrow::struct_({arrow::field("enabled", arrow::boolean()), + arrow::field("event_time", arrow::timestamp(arrow::TimeUnit::MICRO))})); + AssertFieldEqualsJson( + rows, "f_map_row", map_row_type, + R"([[ ["first", [true, "2024-02-29 12:34:56.123456"]], ["second", null] ], null, []])"); + + // Top-level VARIANT values are compared as normalized JSON. + const std::string object_variant_json = + R"({"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}})"; + const std::string array_variant_json = R"([1,"two",false,{"k":"v"}])"; + std::shared_ptr variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_variant")); + ASSERT_TRUE(variants); + AssertVariantJsonAt(variants, 0, object_variant_json); + ASSERT_TRUE(variants->IsNull(1)); + ASSERT_TRUE(variants->IsNull(2)); + + // Deeply nested ROW -> ARRAY> and ROW -> MAP values. + std::shared_ptr deep_rows = + std::dynamic_pointer_cast(rows->GetFieldByName("f_deep_row")); + ASSERT_TRUE(deep_rows); + ASSERT_FALSE(deep_rows->IsNull(0)); + ASSERT_TRUE(deep_rows->IsNull(1)); + ASSERT_TRUE(deep_rows->IsNull(2)); + std::shared_ptr children = + std::dynamic_pointer_cast(deep_rows->GetFieldByName("children")); + ASSERT_TRUE(children); + ASSERT_EQ(children->value_length(0), 2); + std::shared_ptr child_rows = + std::dynamic_pointer_cast(children->values()); + ASSERT_TRUE(child_rows); + AssertArrayEqualsJson(child_rows->GetFieldByName("leaf_id"), "f_deep_row.children.leaf_id", + "[10, 11]"); + std::shared_ptr leaf_variants = + std::dynamic_pointer_cast(child_rows->GetFieldByName("leaf_variant")); + ASSERT_TRUE(leaf_variants); + AssertVariantJsonAt(leaf_variants, 0, array_variant_json); + ASSERT_TRUE(leaf_variants->IsNull(1)); + std::shared_ptr labels = + std::dynamic_pointer_cast(deep_rows->GetFieldByName("labels")); + ASSERT_TRUE(labels); + AssertArrayEqualsJson(labels->Slice(0, 1), "f_deep_row.labels", + R"([[ ["language", "python"], ["format", "parquet"] ]])"); + + // ARRAY values, including null elements and empty arrays. + std::shared_ptr array_variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_array_variant")); + ASSERT_TRUE(array_variants); + ASSERT_EQ(array_variants->value_length(0), 3); + ASSERT_TRUE(array_variants->IsNull(1)); + ASSERT_EQ(array_variants->value_length(2), 0); + std::shared_ptr array_variant_values = + std::dynamic_pointer_cast(array_variants->values()); + ASSERT_TRUE(array_variant_values); + AssertVariantJsonAt(array_variant_values, 0, object_variant_json); + ASSERT_TRUE(array_variant_values->IsNull(1)); + AssertVariantJsonAt(array_variant_values, 2, array_variant_json); + + // MAP values and their key/value alignment. + std::shared_ptr map_variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_map_variant")); + ASSERT_TRUE(map_variants); + ASSERT_EQ(map_variants->value_length(0), 2); + ASSERT_TRUE(map_variants->IsNull(1)); + ASSERT_EQ(map_variants->value_length(2), 0); + AssertArrayEqualsJson(map_variants->keys(), "f_map_variant.keys", R"(["object", "array"])"); + std::shared_ptr map_variant_values = + std::dynamic_pointer_cast(map_variants->items()); + ASSERT_TRUE(map_variant_values); + AssertVariantJsonAt(map_variant_values, 0, object_variant_json); + AssertVariantJsonAt(map_variant_values, 1, array_variant_json); +} + +TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) { + const std::string& writer_prefix = GetParam(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadTable(writer_prefix + "_types", {"f_blob", "f_blob_descriptor"}, + {{Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + std::shared_ptr rows = GetOnlyStructChunk(result); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 3); + ASSERT_EQ(rows->num_fields(), 3); + + // External BLOB descriptors, including a non-empty and an empty blob range. + std::shared_ptr blobs = + std::dynamic_pointer_cast(rows->GetFieldByName("f_blob")); + ASSERT_TRUE(blobs); + ASSERT_EQ(blobs->length(), 3); + ASSERT_TRUE(blobs->IsNull(1)); + std::string_view first = blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_descriptor, + BlobDescriptor::Deserialize(first.data(), first.size())); + ASSERT_EQ(first_descriptor->Version(), 2); + ASSERT_EQ(first_descriptor->Offset(), 4); + ASSERT_EQ(first_descriptor->Length(), 41); + ASSERT_EQ(first_descriptor->Uri().substr(first_descriptor->Uri().size() - 5), ".blob"); + std::string_view empty = blobs->GetView(2); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_descriptor, + BlobDescriptor::Deserialize(empty.data(), empty.size())); + ASSERT_EQ(empty_descriptor->Offset(), 61); + ASSERT_EQ(empty_descriptor->Length(), 0); + + // Resolve the external BLOB and validate its payload instead of its descriptor. + std::map blob_value_options = { + {Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "false"}, + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob_value_result, + ReadTable(writer_prefix + "_types", {"f_blob"}, blob_value_options)); + std::shared_ptr blob_value_rows = GetOnlyStructChunk(blob_value_result); + ASSERT_TRUE(blob_value_rows); + AssertFieldEqualsJson(blob_value_rows, "f_blob", arrow::large_binary(), + R"(["ordinary blob payload from pypaimon 2.0.0", null, ""])"); + + // Inline BLOB descriptors exercise compatible binary types restored from ARROW:schema. + std::shared_ptr inline_blobs = + std::dynamic_pointer_cast( + rows->GetFieldByName("f_blob_descriptor")); + ASSERT_TRUE(inline_blobs); + ASSERT_EQ(inline_blobs->length(), 3); + ASSERT_FALSE(inline_blobs->IsNull(0)); + ASSERT_TRUE(inline_blobs->IsNull(1)); + ASSERT_TRUE(inline_blobs->IsNull(2)); + std::string_view inline_blob = inline_blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr inline_descriptor, + BlobDescriptor::Deserialize(inline_blob.data(), inline_blob.size())); + ASSERT_EQ(inline_descriptor->Version(), 2); + ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); + ASSERT_EQ(inline_descriptor->Offset(), 7); + ASSERT_EQ(inline_descriptor->Length(), 11); +} + +TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) { + const std::string& writer_prefix = GetParam(); + + // Validate every supported VECTOR element type. + std::vector vector_fields = { + "id", "f_vector_boolean", "f_vector_tinyint", "f_vector_smallint", + "f_vector_int", "f_vector_bigint", "f_vector_float", "f_vector_double", + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_result, + ReadTable(writer_prefix + "_vector_types", vector_fields)); + std::shared_ptr vector_rows = GetOnlyStructChunk(vector_result); + ASSERT_TRUE(vector_rows); + ASSERT_EQ(vector_rows->length(), 2); + AssertFieldEqualsJson(vector_rows, "id", arrow::int32(), "[1, 2]"); + AssertFieldEqualsJson(vector_rows, "f_vector_boolean", + arrow::fixed_size_list(arrow::boolean(), 3), + "[[true, false, true], [false, true, false]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_tinyint", arrow::fixed_size_list(arrow::int8(), 3), + "[[-1, 0, 1], [2, 3, 4]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_smallint", + arrow::fixed_size_list(arrow::int16(), 3), + "[[-1000, 0, 1000], [2000, 3000, 4000]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_int", arrow::fixed_size_list(arrow::int32(), 3), + "[[-100000, 0, 100000], [200000, 300000, 400000]]"); + AssertFieldEqualsJson( + vector_rows, "f_vector_bigint", arrow::fixed_size_list(arrow::int64(), 3), + "[[-10000000000, 0, 10000000000], [20000000000, 30000000000, 40000000000]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_float", + arrow::fixed_size_list(arrow::float32(), 3), + "[[1.25, -2.5, 3.75], [4.25, 5.5, 6.75]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_double", + arrow::fixed_size_list(arrow::float64(), 3), + "[[1.125, -2.25, 3.5], [4.125, 5.25, 6.5]]"); +} + +INSTANTIATE_TEST_SUITE_P(Writers, PaimonReadCompatInteTest, ::testing::ValuesIn(WriterPrefixes()), + [](const ::testing::TestParamInfo& info) { + return info.param; + }); + +struct UnsupportedReadCase { + std::string name; + std::string table_suffix; + std::string field_name; + std::string expected_error; +}; + +struct UnsupportedReadParam { + std::string writer_prefix; + UnsupportedReadCase read_case; +}; + +class PaimonUnsupportedTypeInteTest : public ::testing::TestWithParam {}; + +TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError) { + const UnsupportedReadParam& test_case = GetParam(); + ASSERT_NOK_WITH_MSG(ReadTable(test_case.writer_prefix + "_" + test_case.read_case.table_suffix, + {test_case.read_case.field_name}), + test_case.read_case.expected_error); +} + +std::vector UnsupportedReadParams() { + const std::vector read_cases = { + {"NestedBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, + {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, + {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, + {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, + {"TimePrecision9", "time_types", "f_time_9", "Unsupported type: TIME"}, + }; + std::vector result; + for (const std::string& writer_prefix : WriterPrefixes()) { + for (const UnsupportedReadCase& read_case : read_cases) { + result.push_back({writer_prefix, read_case}); + } + } + return result; +} + +INSTANTIATE_TEST_SUITE_P(UnsupportedTypes, PaimonUnsupportedTypeInteTest, + ::testing::ValuesIn(UnsupportedReadParams()), + [](const ::testing::TestParamInfo& info) { + return info.param.writer_prefix + info.param.read_case.name; + }); + +} // namespace paimon::test diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md b/test/test_data/parquet/append_types_compatibility.db/README.md new file mode 100644 index 000000000..5e4e330be --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/README.md @@ -0,0 +1,45 @@ + + +# Paimon writer compatibility fixtures + +The `append_types_compatibility.db` database contains equivalent append-only tables written by Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of types in that table; no individual table is claimed to contain every Paimon type. + +The writers and Parquet implementations are: + +- `python_*`: `pypaimon==2.0.0` with `pyarrow==19.0.1`. The Parquet files contain `ARROW:schema` metadata. +- `rust_*`: Paimon Rust 0.4.0 at commit `ced0c86b4db76265b5b2aedcbc9909a208f2c130`, using `parquet-rs version 58.4.0`. The Parquet files contain `ARROW:schema` metadata. +- `java_*`: the Java batch-write API from `paimon-bundle-2-ali-2.6.jar`, using `parquet-mr version 1.16.0`. The Parquet files do not contain `ARROW:schema` metadata. + +## Table groups + +Each writer prefix has four tables with the same schema and logical values: + +- `_types`: 43 columns and 3 rows covering integral and floating types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE, TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions, VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in every nullable column and row 3 exercises empty values. +- `_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, and DOUBLE. +- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ rejects the table schema because ARRAY<BLOB> is unsupported. Schema validation happens before column projection, so the compatibility test explicitly checks this error instead of reading either field. +- `_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++ currently rejects `TIME` while parsing the table schema. + +The tables are separated because Paimon C++ does not allow VECTOR in a data-evolution table, BLOB requires data evolution, and a schema-level incompatibility must not prevent compatible columns from being tested. + +`MULTISET` is absent because the Python batch writer cannot produce it through its PyArrow conversion and Paimon C++ rejects it while parsing the table schema. Paimon Rust 0.4.0 supports `MULTISET` and represents it as an Arrow map from each element to its `INT` count, but it cannot be included in an equivalent three-writer fixture. Consequently, this fixture is a compatibility matrix, not an assertion that every type supported by every Paimon implementation can be represented by one table. + +The `f_blob_descriptor` value intentionally points at a nonexistent external URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor and Parquet `ARROW:schema` path rather than external blob fetching. + +Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types differ in these fixtures. The Python fixture records the physical Parquet `binary` column as Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test verifies that Paimon C++ can read both representations and preserve the descriptor value. diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet new file mode 100644 index 000000000..2d2cb3152 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob new file mode 100644 index 000000000..6bf912ee2 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob new file mode 100644 index 000000000..bf82127da Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 new file mode 100644 index 000000000..f12e5f846 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 new file mode 100644 index 000000000..034d68e17 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 new file mode 100644 index 000000000..071447210 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 new file mode 100644 index 000000000..78945df1c --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_array_blob", + "type" : { + "type" : "ARRAY", + "element" : "BLOB" + } + }, { + "id" : 2, + "name" : "f_map_blob", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "BLOB" + } + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "blob-field" : "f_array_blob,f_map_blob", + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788853436235 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..b98c7d14d --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "89919eb0-8f08-4098-9002-d19680ca3a6d", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1", + "deltaManifestListSize" : 1111, + "commitUser" : "1192f15e-31f1-42a4-b498-248a36a42985", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436256, + "totalRecordCount" : 9, + "deltaRecordCount" : 9, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/bucket-0/data-66214e22-ae63-42cf-8971-37326665ac74-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_time_types/bucket-0/data-66214e22-ae63-42cf-8971-37326665ac74-0.parquet new file mode 100644 index 000000000..80cc347ec Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_time_types/bucket-0/data-66214e22-ae63-42cf-8971-37326665ac74-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-fc7846ba-bc50-4067-86a4-d70bbc0d979f-0 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-fc7846ba-bc50-4067-86a4-d70bbc0d979f-0 new file mode 100644 index 000000000..661825c96 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-fc7846ba-bc50-4067-86a4-d70bbc0d979f-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0 new file mode 100644 index 000000000..c249e4b77 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 new file mode 100644 index 000000000..c0a0585fe Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 new file mode 100644 index 000000000..02f0fbffb --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 @@ -0,0 +1,34 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_time_0", + "type" : "TIME(0)" + }, { + "id" : 2, + "name" : "f_time_3", + "type" : "TIME(3)" + }, { + "id" : 3, + "name" : "f_time_6", + "type" : "TIME(6)" + }, { + "id" : 4, + "name" : "f_time_9", + "type" : "TIME(9)" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788853436258 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 new file mode 100644 index 000000000..651a6a907 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40ea5cbe-9f6a-463c-a506-4b8a21041b6d", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1", + "deltaManifestListSize" : 1108, + "commitUser" : "3adee06a-0e07-431a-aaa3-762caba0f779", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436279, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet new file mode 100644 index 000000000..aa712022f Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-1.blob b/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-1.blob new file mode 100644 index 000000000..6561d11da Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-1.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-92f63f4a-170d-4a74-a9e2-364bdb2c1e52-0 b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-92f63f4a-170d-4a74-a9e2-364bdb2c1e52-0 new file mode 100644 index 000000000..ffc77cdda Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-92f63f4a-170d-4a74-a9e2-364bdb2c1e52-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0 b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0 new file mode 100644 index 000000000..17a9d48bd Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 new file mode 100644 index 000000000..f62987afd Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 new file mode 100644 index 000000000..554c947a9 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 @@ -0,0 +1,306 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_boolean", + "type" : "BOOLEAN" + }, { + "id" : 2, + "name" : "f_tinyint", + "type" : "TINYINT" + }, { + "id" : 3, + "name" : "f_smallint", + "type" : "SMALLINT" + }, { + "id" : 4, + "name" : "f_int", + "type" : "INT" + }, { + "id" : 5, + "name" : "f_bigint", + "type" : "BIGINT" + }, { + "id" : 6, + "name" : "f_float", + "type" : "FLOAT" + }, { + "id" : 7, + "name" : "f_double", + "type" : "DOUBLE" + }, { + "id" : 8, + "name" : "f_char", + "type" : "CHAR(8)" + }, { + "id" : 9, + "name" : "f_varchar", + "type" : "VARCHAR(64)" + }, { + "id" : 10, + "name" : "f_string", + "type" : "STRING" + }, { + "id" : 11, + "name" : "f_binary", + "type" : "BINARY(8)" + }, { + "id" : 12, + "name" : "f_varbinary", + "type" : "VARBINARY(64)" + }, { + "id" : 13, + "name" : "f_bytes", + "type" : "BYTES" + }, { + "id" : 14, + "name" : "f_blob", + "type" : "BLOB" + }, { + "id" : 15, + "name" : "f_blob_descriptor", + "type" : "BLOB" + }, { + "id" : 16, + "name" : "f_decimal_1_0", + "type" : "DECIMAL(1, 0)" + }, { + "id" : 17, + "name" : "f_decimal_9_2", + "type" : "DECIMAL(9, 2)" + }, { + "id" : 18, + "name" : "f_decimal_18_2", + "type" : "DECIMAL(18, 2)" + }, { + "id" : 19, + "name" : "f_decimal_19_2", + "type" : "DECIMAL(19, 2)" + }, { + "id" : 20, + "name" : "f_decimal_38_18", + "type" : "DECIMAL(38, 18)" + }, { + "id" : 21, + "name" : "f_decimal_38_38", + "type" : "DECIMAL(38, 38)" + }, { + "id" : 22, + "name" : "f_date", + "type" : "DATE" + }, { + "id" : 23, + "name" : "f_timestamp_0", + "type" : "TIMESTAMP(0)" + }, { + "id" : 24, + "name" : "f_timestamp_3", + "type" : "TIMESTAMP(3)" + }, { + "id" : 25, + "name" : "f_timestamp_6", + "type" : "TIMESTAMP(6)" + }, { + "id" : 26, + "name" : "f_timestamp_9", + "type" : "TIMESTAMP(9)" + }, { + "id" : 27, + "name" : "f_timestamp_ltz_0", + "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE" + }, { + "id" : 28, + "name" : "f_timestamp_ltz_3", + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, { + "id" : 29, + "name" : "f_timestamp_ltz_6", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + }, { + "id" : 30, + "name" : "f_timestamp_ltz_9", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + }, { + "id" : 31, + "name" : "f_variant", + "type" : "VARIANT" + }, { + "id" : 32, + "name" : "f_array_int", + "type" : { + "type" : "ARRAY", + "element" : "INT" + } + }, { + "id" : 33, + "name" : "f_map_string_bigint", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "BIGINT" + } + }, { + "id" : 34, + "name" : "f_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 35, + "name" : "nested_int", + "type" : "INT" + }, { + "id" : 36, + "name" : "nested_string", + "type" : "STRING" + }, { + "id" : 37, + "name" : "nested_decimal", + "type" : "DECIMAL(19, 4)" + }, { + "id" : 38, + "name" : "nested_timestamp", + "type" : "TIMESTAMP(9)" + }, { + "id" : 39, + "name" : "nested_timestamp_ltz", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + } ] + } + }, { + "id" : 40, + "name" : "f_array_array_int", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ARRAY", + "element" : "INT" + } + } + }, { + "id" : 41, + "name" : "f_array_map", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "INT" + } + } + }, { + "id" : 42, + "name" : "f_map_array", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : { + "type" : "ARRAY", + "element" : "INT" + } + } + }, { + "id" : 43, + "name" : "f_array_row", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ROW", + "fields" : [ { + "id" : 44, + "name" : "name", + "type" : "STRING" + }, { + "id" : 45, + "name" : "score", + "type" : "DECIMAL(9, 2)" + } ] + } + } + }, { + "id" : 46, + "name" : "f_map_row", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : { + "type" : "ROW", + "fields" : [ { + "id" : 47, + "name" : "enabled", + "type" : "BOOLEAN" + }, { + "id" : 48, + "name" : "event_time", + "type" : "TIMESTAMP(6)" + } ] + } + } + }, { + "id" : 49, + "name" : "f_deep_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 52, + "name" : "children", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ROW", + "fields" : [ { + "id" : 50, + "name" : "leaf_id", + "type" : "BIGINT" + }, { + "id" : 51, + "name" : "leaf_variant", + "type" : "VARIANT" + } ] + } + } + }, { + "id" : 53, + "name" : "labels", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "STRING" + } + } ] + } + }, { + "id" : 54, + "name" : "f_array_variant", + "type" : { + "type" : "ARRAY", + "element" : "VARIANT" + } + }, { + "id" : 55, + "name" : "f_map_variant", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "VARIANT" + } + } ], + "highestFieldId" : 55, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "blob-field" : "f_blob,f_blob_descriptor", + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "avro", + "file.format" : "parquet", + "blob-descriptor-field" : "f_blob_descriptor" + }, + "comment" : "All Paimon data types written by Java Paimon", + "timeMillis" : 1788853434699 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 new file mode 100644 index 000000000..6483d0d61 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "b90d1c93-48c8-4f97-9201-484b9c9192d3", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1", + "deltaManifestListSize" : 1115, + "commitUser" : "2ec0ad8a-b06e-46ff-ac6c-270da2027e88", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436130, + "totalRecordCount" : 6, + "deltaRecordCount" : 6, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet new file mode 100644 index 000000000..c850af24c Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 new file mode 100644 index 000000000..85f300079 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 new file mode 100644 index 000000000..2f4d7efd1 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1 new file mode 100644 index 000000000..a540f30be Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 new file mode 100644 index 000000000..6e93d2b42 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 @@ -0,0 +1,74 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_vector_boolean", + "type" : { + "type" : "VECTOR", + "element" : "BOOLEAN", + "length" : 3 + } + }, { + "id" : 2, + "name" : "f_vector_tinyint", + "type" : { + "type" : "VECTOR", + "element" : "TINYINT", + "length" : 3 + } + }, { + "id" : 3, + "name" : "f_vector_smallint", + "type" : { + "type" : "VECTOR", + "element" : "SMALLINT", + "length" : 3 + } + }, { + "id" : 4, + "name" : "f_vector_int", + "type" : { + "type" : "VECTOR", + "element" : "INT", + "length" : 3 + } + }, { + "id" : 5, + "name" : "f_vector_bigint", + "type" : { + "type" : "VECTOR", + "element" : "BIGINT", + "length" : 3 + } + }, { + "id" : 6, + "name" : "f_vector_float", + "type" : { + "type" : "VECTOR", + "element" : "FLOAT", + "length" : 3 + } + }, { + "id" : 7, + "name" : "f_vector_double", + "type" : { + "type" : "VECTOR", + "element" : "DOUBLE", + "length" : 3 + } + } ], + "highestFieldId" : 7, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788853436194 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 new file mode 100644 index 000000000..e07a54596 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "0631a4c4-fe4a-420c-b6e8-8dee879154c9", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1", + "deltaManifestListSize" : 1108, + "commitUser" : "460bb465-4bf5-4692-92df-5c54b4d910dd", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436225, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet new file mode 100644 index 000000000..d733aa4d5 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob new file mode 100644 index 000000000..bf82127da Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob new file mode 100644 index 000000000..1ec653402 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 new file mode 100644 index 000000000..eb9d186f6 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 new file mode 100644 index 000000000..8608500a4 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 new file mode 100644 index 000000000..c85847cb0 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 new file mode 100644 index 000000000..e311af948 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 @@ -0,0 +1,43 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_array_blob", + "type": { + "type": "ARRAY", + "element": "BLOB", + "nullable": true + } + }, + { + "id": 2, + "name": "f_map_blob", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "BLOB", + "nullable": true + } + } + ], + "highestFieldId": 2, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro", + "row-tracking.enabled": "true", + "data-evolution.enabled": "true", + "blob-field": "f_array_blob,f_map_blob" + }, + "comment": "Generated by pypaimon 2.0.0 with ARRAY and MAP.", + "timeMillis": 1788513128455 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..8e955da73 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0", + "deltaManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1", + "totalRecordCount": 9, + "deltaRecordCount": 9, + "commitUser": "297eeaeb-5c62-4b3c-8f9e-950b09612bc3", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128459, + "nextRowId": 3, + "uuid": "d2837b32-27c3-453e-8470-ccb9d176d3f0" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet new file mode 100644 index 000000000..e689b55e1 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 new file mode 100644 index 000000000..2a42c2af5 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0 new file mode 100644 index 000000000..946897252 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 new file mode 100644 index 000000000..f763a89f9 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0 new file mode 100644 index 000000000..99509c462 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0 @@ -0,0 +1,41 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_time_0", + "type": "TIME(0)" + }, + { + "id": 2, + "name": "f_time_3", + "type": "TIME(3)" + }, + { + "id": 3, + "name": "f_time_6", + "type": "TIME(6)" + }, + { + "id": 4, + "name": "f_time_9", + "type": "TIME(9)" + } + ], + "highestFieldId": 4, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro" + }, + "comment": "Generated by pypaimon 2.0.0 with Python-supported TIME declarations.", + "timeMillis": 1788513128460 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/snapshot-1 new file mode 100644 index 000000000..599987a66 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/snapshot-1 @@ -0,0 +1,14 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0", + "deltaManifestList": "manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1", + "totalRecordCount": 2, + "deltaRecordCount": 2, + "commitUser": "8b834eed-3798-4cd0-b2d6-06b073ef6cb0", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128463, + "uuid": "98db2907-43d1-455f-85b1-928e587b8816" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-37d6e1a8-ff91-4765-b6a8-e6db2f3c008b-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-37d6e1a8-ff91-4765-b6a8-e6db2f3c008b-0.blob new file mode 100644 index 000000000..d0994f305 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-37d6e1a8-ff91-4765-b6a8-e6db2f3c008b-0.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet new file mode 100644 index 000000000..bc1c0ee2b Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 new file mode 100644 index 000000000..5bf90fd66 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0 b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0 new file mode 100644 index 000000000..0aab6f2f4 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 new file mode 100644 index 000000000..d004f2bd8 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0 new file mode 100644 index 000000000..ccffbb673 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0 @@ -0,0 +1,387 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_boolean", + "type": "BOOLEAN" + }, + { + "id": 2, + "name": "f_tinyint", + "type": "TINYINT" + }, + { + "id": 3, + "name": "f_smallint", + "type": "SMALLINT" + }, + { + "id": 4, + "name": "f_int", + "type": "INT" + }, + { + "id": 5, + "name": "f_bigint", + "type": "BIGINT" + }, + { + "id": 6, + "name": "f_float", + "type": "FLOAT" + }, + { + "id": 7, + "name": "f_double", + "type": "DOUBLE" + }, + { + "id": 8, + "name": "f_char", + "type": "CHAR(8)" + }, + { + "id": 9, + "name": "f_varchar", + "type": "VARCHAR(64)" + }, + { + "id": 10, + "name": "f_string", + "type": "STRING" + }, + { + "id": 11, + "name": "f_binary", + "type": "BINARY(8)" + }, + { + "id": 12, + "name": "f_varbinary", + "type": "VARBINARY(64)" + }, + { + "id": 13, + "name": "f_bytes", + "type": "BYTES" + }, + { + "id": 14, + "name": "f_blob", + "type": "BLOB" + }, + { + "id": 15, + "name": "f_blob_descriptor", + "type": "BLOB" + }, + { + "id": 16, + "name": "f_decimal_1_0", + "type": "DECIMAL(1, 0)" + }, + { + "id": 17, + "name": "f_decimal_9_2", + "type": "DECIMAL(9, 2)" + }, + { + "id": 18, + "name": "f_decimal_18_2", + "type": "DECIMAL(18, 2)" + }, + { + "id": 19, + "name": "f_decimal_19_2", + "type": "DECIMAL(19, 2)" + }, + { + "id": 20, + "name": "f_decimal_38_18", + "type": "DECIMAL(38, 18)" + }, + { + "id": 21, + "name": "f_decimal_38_38", + "type": "DECIMAL(38, 38)" + }, + { + "id": 22, + "name": "f_date", + "type": "DATE" + }, + { + "id": 23, + "name": "f_timestamp_0", + "type": "TIMESTAMP(0)" + }, + { + "id": 24, + "name": "f_timestamp_3", + "type": "TIMESTAMP(3)" + }, + { + "id": 25, + "name": "f_timestamp_6", + "type": "TIMESTAMP(6)" + }, + { + "id": 26, + "name": "f_timestamp_9", + "type": "TIMESTAMP(9)" + }, + { + "id": 27, + "name": "f_timestamp_ltz_0", + "type": "TIMESTAMP_LTZ(0)" + }, + { + "id": 28, + "name": "f_timestamp_ltz_3", + "type": "TIMESTAMP_LTZ(3)" + }, + { + "id": 29, + "name": "f_timestamp_ltz_6", + "type": "TIMESTAMP_LTZ(6)" + }, + { + "id": 30, + "name": "f_timestamp_ltz_9", + "type": "TIMESTAMP_LTZ(9)" + }, + { + "id": 31, + "name": "f_variant", + "type": "VARIANT" + }, + { + "id": 32, + "name": "f_array_int", + "type": { + "type": "ARRAY", + "element": "INT", + "nullable": true + } + }, + { + "id": 33, + "name": "f_map_string_bigint", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "BIGINT", + "nullable": true + } + }, + { + "id": 39, + "name": "f_row", + "type": { + "type": "ROW", + "fields": [ + { + "id": 34, + "name": "nested_int", + "type": "INT" + }, + { + "id": 35, + "name": "nested_string", + "type": "STRING" + }, + { + "id": 36, + "name": "nested_decimal", + "type": "DECIMAL(19, 4)" + }, + { + "id": 37, + "name": "nested_timestamp", + "type": "TIMESTAMP(9)" + }, + { + "id": 38, + "name": "nested_timestamp_ltz", + "type": "TIMESTAMP_LTZ(9)" + } + ], + "nullable": true + } + }, + { + "id": 40, + "name": "f_array_array_int", + "type": { + "type": "ARRAY", + "element": { + "type": "ARRAY", + "element": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 41, + "name": "f_array_map", + "type": { + "type": "ARRAY", + "element": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 42, + "name": "f_map_array", + "type": { + "type": "MAP>", + "key": "STRING NOT NULL", + "value": { + "type": "ARRAY", + "element": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 45, + "name": "f_array_row", + "type": { + "type": "ARRAY", + "element": { + "type": "ROW", + "fields": [ + { + "id": 43, + "name": "name", + "type": "STRING" + }, + { + "id": 44, + "name": "score", + "type": "DECIMAL(9, 2)" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 48, + "name": "f_map_row", + "type": { + "type": "MAP>", + "key": "STRING NOT NULL", + "value": { + "type": "ROW", + "fields": [ + { + "id": 46, + "name": "enabled", + "type": "BOOLEAN" + }, + { + "id": 47, + "name": "event_time", + "type": "TIMESTAMP(6)" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 53, + "name": "f_deep_row", + "type": { + "type": "ROW", + "fields": [ + { + "id": 51, + "name": "children", + "type": { + "type": "ARRAY", + "element": { + "type": "ROW", + "fields": [ + { + "id": 49, + "name": "leaf_id", + "type": "BIGINT" + }, + { + "id": 50, + "name": "leaf_variant", + "type": "VARIANT" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 52, + "name": "labels", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "STRING", + "nullable": true + } + } + ], + "nullable": true + } + }, + { + "id": 54, + "name": "f_array_variant", + "type": { + "type": "ARRAY", + "element": "VARIANT", + "nullable": true + } + }, + { + "id": 55, + "name": "f_map_variant", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "VARIANT", + "nullable": true + } + } + ], + "highestFieldId": 55, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro", + "row-tracking.enabled": "true", + "data-evolution.enabled": "true", + "blob-field": "f_blob,f_blob_descriptor", + "blob-descriptor-field": "f_blob_descriptor" + }, + "comment": "Generated by pypaimon 2.0.0 with scalar, blob, variant, and nested types.", + "timeMillis": 1788513128436 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 new file mode 100644 index 000000000..4efdf367b --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0", + "deltaManifestList": "manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "4660bd51-f9dd-40a1-9408-3c83d48e2336", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128449, + "nextRowId": 3, + "uuid": "e60f3846-91eb-435f-8ef9-161ace2622ae" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet new file mode 100644 index 000000000..d39e4856b Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-c830fb42-bbd0-416c-9349-d01b97497d7b-0 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-c830fb42-bbd0-416c-9349-d01b97497d7b-0 new file mode 100644 index 000000000..105316f00 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-c830fb42-bbd0-416c-9349-d01b97497d7b-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 new file mode 100644 index 000000000..55c490337 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 new file mode 100644 index 000000000..3dc50f7c3 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0 new file mode 100644 index 000000000..f7cf4b495 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0 @@ -0,0 +1,91 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_vector_boolean", + "type": { + "type": "VECTOR", + "element": "BOOLEAN", + "length": 3, + "nullable": true + } + }, + { + "id": 2, + "name": "f_vector_tinyint", + "type": { + "type": "VECTOR", + "element": "TINYINT", + "length": 3, + "nullable": true + } + }, + { + "id": 3, + "name": "f_vector_smallint", + "type": { + "type": "VECTOR", + "element": "SMALLINT", + "length": 3, + "nullable": true + } + }, + { + "id": 4, + "name": "f_vector_int", + "type": { + "type": "VECTOR", + "element": "INT", + "length": 3, + "nullable": true + } + }, + { + "id": 5, + "name": "f_vector_bigint", + "type": { + "type": "VECTOR", + "element": "BIGINT", + "length": 3, + "nullable": true + } + }, + { + "id": 6, + "name": "f_vector_float", + "type": { + "type": "VECTOR", + "element": "FLOAT", + "length": 3, + "nullable": true + } + }, + { + "id": 7, + "name": "f_vector_double", + "type": { + "type": "VECTOR", + "element": "DOUBLE", + "length": 3, + "nullable": true + } + } + ], + "highestFieldId": 7, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro" + }, + "comment": "Generated by pypaimon 2.0.0 with every supported VECTOR element type.", + "timeMillis": 1788513128451 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/snapshot-1 new file mode 100644 index 000000000..d4a55eabf --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/snapshot-1 @@ -0,0 +1,14 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0", + "deltaManifestList": "manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1", + "totalRecordCount": 2, + "deltaRecordCount": 2, + "commitUser": "eb6e21e8-3127-4ab5-b4ad-1cc3a99cb1a2", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128454, + "uuid": "0c0e3f65-06ad-4d58-9f58-a49dbf8ead09" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet new file mode 100644 index 000000000..0383ee56b Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 new file mode 100644 index 000000000..286b07f38 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 new file mode 100644 index 000000000..a01a7456a Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 new file mode 100644 index 000000000..5478f74d4 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 new file mode 100644 index 000000000..66ceca0a3 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}},{"id":2,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT NULL","value":"BLOB"}}],"highestFieldId":2,"partitionKeys":[],"primaryKeys":[],"options":{"manifest.format":"avro","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1","row-tracking.enabled":"true","blob-field":"f_array_blob,f_map_blob"},"comment":null,"timeMillis":1788858155021} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..c24110af8 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0","deltaManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1","commitUser":"ce789997-7558-44de-8068-c5d7e744dba9","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155032,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/bucket-0/data-08a933f1-9c3f-4cbc-9a9b-dcc743380316-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/bucket-0/data-08a933f1-9c3f-4cbc-9a9b-dcc743380316-0.parquet new file mode 100644 index 000000000..cc59010e4 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/bucket-0/data-08a933f1-9c3f-4cbc-9a9b-dcc743380316-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 new file mode 100644 index 000000000..8a8baf5fb Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 new file mode 100644 index 000000000..eb1752a53 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-1 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-1 new file mode 100644 index 000000000..a4b839c28 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0 new file mode 100644 index 000000000..dccabf94f --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_time_0","type":"TIME(0)"},{"id":2,"name":"f_time_3","type":"TIME(3)"},{"id":3,"name":"f_time_6","type":"TIME(6)"},{"id":4,"name":"f_time_9","type":"TIME(9)"}],"highestFieldId":4,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","bucket":"-1","manifest.format":"avro"},"comment":null,"timeMillis":1788858155034} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/snapshot-1 new file mode 100644 index 000000000..fbbbb601a --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0","deltaManifestList":"manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-1","commitUser":"8ecf53a7-46c4-4636-aafa-e4e40992149b","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155043,"totalRecordCount":2,"deltaRecordCount":2} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-5b3012c1-f361-49d3-be0a-51dfc4777014-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-5b3012c1-f361-49d3-be0a-51dfc4777014-0.parquet new file mode 100644 index 000000000..4c510df39 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-5b3012c1-f361-49d3-be0a-51dfc4777014-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-ada58573-1b72-4849-86fa-8caca9413789-0.blob b/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-ada58573-1b72-4849-86fa-8caca9413789-0.blob new file mode 100644 index 000000000..6561d11da Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-ada58573-1b72-4849-86fa-8caca9413789-0.blob differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-93f766c3-ee4f-4638-9cf3-823281ee15e4-0 b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-93f766c3-ee4f-4638-9cf3-823281ee15e4-0 new file mode 100644 index 000000000..273dd8d57 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-93f766c3-ee4f-4638-9cf3-823281ee15e4-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0 b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0 new file mode 100644 index 000000000..f8f03ae61 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1 b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1 new file mode 100644 index 000000000..09ee0b15b Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 new file mode 100644 index 000000000..8118fd305 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_boolean","type":"BOOLEAN"},{"id":2,"name":"f_tinyint","type":"TINYINT"},{"id":3,"name":"f_smallint","type":"SMALLINT"},{"id":4,"name":"f_int","type":"INT"},{"id":5,"name":"f_bigint","type":"BIGINT"},{"id":6,"name":"f_float","type":"FLOAT"},{"id":7,"name":"f_double","type":"DOUBLE"},{"id":8,"name":"f_char","type":"CHAR(8)"},{"id":9,"name":"f_varchar","type":"VARCHAR(64)"},{"id":10,"name":"f_string","type":"STRING"},{"id":11,"name":"f_binary","type":"BINARY(8)"},{"id":12,"name":"f_varbinary","type":"VARBINARY(64)"},{"id":13,"name":"f_bytes","type":"VARBINARY(2147483647)"},{"id":14,"name":"f_blob","type":"BLOB"},{"id":15,"name":"f_blob_descriptor","type":"BLOB"},{"id":16,"name":"f_decimal_1_0","type":"DECIMAL(1, 0)"},{"id":17,"name":"f_decimal_9_2","type":"DECIMAL(9, 2)"},{"id":18,"name":"f_decimal_18_2","type":"DECIMAL(18, 2)"},{"id":19,"name":"f_decimal_19_2","type":"DECIMAL(19, 2)"},{"id":20,"name":"f_decimal_38_18","type":"DECIMAL(38, 18)"},{"id":21,"name":"f_decimal_38_38","type":"DECIMAL(38, 38)"},{"id":22,"name":"f_date","type":"DATE"},{"id":23,"name":"f_timestamp_0","type":"TIMESTAMP(0)"},{"id":24,"name":"f_timestamp_3","type":"TIMESTAMP(3)"},{"id":25,"name":"f_timestamp_6","type":"TIMESTAMP(6)"},{"id":26,"name":"f_timestamp_9","type":"TIMESTAMP(9)"},{"id":27,"name":"f_timestamp_ltz_0","type":"TIMESTAMP(0) WITH LOCAL TIME ZONE"},{"id":28,"name":"f_timestamp_ltz_3","type":"TIMESTAMP(3) WITH LOCAL TIME ZONE"},{"id":29,"name":"f_timestamp_ltz_6","type":"TIMESTAMP(6) WITH LOCAL TIME ZONE"},{"id":30,"name":"f_timestamp_ltz_9","type":"TIMESTAMP(9) WITH LOCAL TIME ZONE"},{"id":31,"name":"f_variant","type":"VARIANT"},{"id":32,"name":"f_array_int","type":{"type":"ARRAY","element":"INT"}},{"id":33,"name":"f_map_string_bigint","type":{"type":"MAP","key":"STRING NOT NULL","value":"BIGINT"}},{"id":34,"name":"f_row","type":{"type":"ROW","fields":[{"id":35,"name":"nested_int","type":"INT"},{"id":36,"name":"nested_string","type":"STRING"},{"id":37,"name":"nested_decimal","type":"DECIMAL(19, 4)"},{"id":38,"name":"nested_timestamp","type":"TIMESTAMP(9)"},{"id":39,"name":"nested_timestamp_ltz","type":"TIMESTAMP(9) WITH LOCAL TIME ZONE"}]}},{"id":40,"name":"f_array_array_int","type":{"type":"ARRAY","element":{"type":"ARRAY","element":"INT"}}},{"id":41,"name":"f_array_map","type":{"type":"ARRAY","element":{"type":"MAP","key":"STRING NOT NULL","value":"INT"}}},{"id":42,"name":"f_map_array","type":{"type":"MAP","key":"STRING NOT NULL","value":{"type":"ARRAY","element":"INT"}}},{"id":43,"name":"f_array_row","type":{"type":"ARRAY","element":{"type":"ROW","fields":[{"id":44,"name":"name","type":"STRING"},{"id":45,"name":"score","type":"DECIMAL(9, 2)"}]}}},{"id":46,"name":"f_map_row","type":{"type":"MAP","key":"STRING NOT NULL","value":{"type":"ROW","fields":[{"id":47,"name":"enabled","type":"BOOLEAN"},{"id":48,"name":"event_time","type":"TIMESTAMP(6)"}]}}},{"id":49,"name":"f_deep_row","type":{"type":"ROW","fields":[{"id":50,"name":"children","type":{"type":"ARRAY","element":{"type":"ROW","fields":[{"id":51,"name":"leaf_id","type":"BIGINT"},{"id":52,"name":"leaf_variant","type":"VARIANT"}]}}},{"id":53,"name":"labels","type":{"type":"MAP","key":"STRING NOT NULL","value":"STRING"}}]}},{"id":54,"name":"f_array_variant","type":{"type":"ARRAY","element":"VARIANT"}},{"id":55,"name":"f_map_variant","type":{"type":"MAP","key":"STRING NOT NULL","value":"VARIANT"}}],"highestFieldId":55,"partitionKeys":[],"primaryKeys":[],"options":{"manifest.format":"avro","blob-field":"f_blob,f_blob_descriptor","blob-descriptor-field":"f_blob_descriptor","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1","row-tracking.enabled":"true"},"comment":"Paimon type compatibility data written by Rust Paimon","timeMillis":1788858154980} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 new file mode 100644 index 000000000..93ff344a5 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0","deltaManifestList":"manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1","commitUser":"12acc040-0373-49d8-9406-4dcc86ea02cf","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155006,"totalRecordCount":6,"deltaRecordCount":6,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet new file mode 100644 index 000000000..ad2eefb5d Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-e53c18e4-2a6c-479f-a696-fbb444f8df62-0 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-e53c18e4-2a6c-479f-a696-fbb444f8df62-0 new file mode 100644 index 000000000..627fd15e1 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-e53c18e4-2a6c-479f-a696-fbb444f8df62-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 new file mode 100644 index 000000000..018d7237f Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1 new file mode 100644 index 000000000..afdf6a69c Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 new file mode 100644 index 000000000..3ee403b98 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_vector_boolean","type":{"type":"VECTOR","element":"BOOLEAN","length":3}},{"id":2,"name":"f_vector_tinyint","type":{"type":"VECTOR","element":"TINYINT","length":3}},{"id":3,"name":"f_vector_smallint","type":{"type":"VECTOR","element":"SMALLINT","length":3}},{"id":4,"name":"f_vector_int","type":{"type":"VECTOR","element":"INT","length":3}},{"id":5,"name":"f_vector_bigint","type":{"type":"VECTOR","element":"BIGINT","length":3}},{"id":6,"name":"f_vector_float","type":{"type":"VECTOR","element":"FLOAT","length":3}},{"id":7,"name":"f_vector_double","type":{"type":"VECTOR","element":"DOUBLE","length":3}}],"highestFieldId":7,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","bucket":"-1"},"comment":null,"timeMillis":1788858155008} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 new file mode 100644 index 000000000..e0535a0c0 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0","deltaManifestList":"manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1","commitUser":"8b326938-c073-4993-bd3e-4761ca4e3e0d","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155020,"totalRecordCount":2,"deltaRecordCount":2} \ No newline at end of file