From 614dffd0ac02d55af8cec3d3be40fa7489fc247f Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Tue, 8 Sep 2026 17:18:45 +0800 Subject: [PATCH 1/3] fix(parquet): handle blobs restored from Arrow schema --- src/paimon/core/io/field_mapping_reader.cpp | 26 +- .../core/io/field_mapping_reader_test.cpp | 46 ++ .../parquet/parquet_timestamp_converter.cpp | 25 +- .../parquet_timestamp_converter_test.cpp | 19 + test/inte/CMakeLists.txt | 7 + test/inte/paimon_read_compat_inte_test.cpp | 500 ++++++++++++++++++ .../append_types_compatibility.db/README.md | 45 ++ ...84c4-ca25-4b99-a024-07dc9526b5dc-0.parquet | Bin 0 -> 298 bytes ...8f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob | Bin 0 -> 93 bytes ...8f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob | Bin 0 -> 100 bytes ...est-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 | Bin 0 -> 2249 bytes ...ist-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 | Bin 0 -> 1006 bytes ...ist-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 | Bin 0 -> 1111 bytes .../java_nested_blob_types/schema/schema-0 | 36 ++ .../java_nested_blob_types/snapshot/EARLIEST | 1 + .../java_nested_blob_types/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 18 + ...4e22-ae63-42cf-8971-37326665ac74-0.parquet | Bin 0 -> 1066 bytes ...est-fc7846ba-bc50-4067-86a4-d70bbc0d979f-0 | Bin 0 -> 2200 bytes ...ist-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0 | Bin 0 -> 1006 bytes ...ist-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 | Bin 0 -> 1108 bytes .../java_time_types/schema/schema-0 | 34 ++ .../java_time_types/snapshot/EARLIEST | 1 + .../java_time_types/snapshot/LATEST | 1 + .../java_time_types/snapshot/snapshot-1 | 18 + ...c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet | Bin 0 -> 16680 bytes ...9e2c4dd-e99e-4a94-831e-a2261e62fbbc-1.blob | Bin 0 -> 81 bytes ...est-92f63f4a-170d-4a74-a9e2-364bdb2c1e52-0 | Bin 0 -> 2802 bytes ...ist-caac218e-853c-4ee6-94a3-2a2b509bfdee-0 | Bin 0 -> 1006 bytes ...ist-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 | Bin 0 -> 1115 bytes .../java_types/schema/schema-0 | 306 +++++++++++ .../java_types/snapshot/EARLIEST | 1 + .../java_types/snapshot/LATEST | 1 + .../java_types/snapshot/snapshot-1 | 18 + ...3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet | Bin 0 -> 2416 bytes ...est-30233ec1-2b71-4793-8005-99a4aa0df863-0 | Bin 0 -> 2189 bytes ...ist-5a2523ff-cfae-47c7-8bec-c476b104f406-0 | Bin 0 -> 1006 bytes ...ist-5a2523ff-cfae-47c7-8bec-c476b104f406-1 | Bin 0 -> 1108 bytes .../java_vector_types/schema/schema-0 | 74 +++ .../java_vector_types/snapshot/EARLIEST | 1 + .../java_vector_types/snapshot/LATEST | 1 + .../java_vector_types/snapshot/snapshot-1 | 18 + ...8c74-bd24-4106-9ed5-91d20d711270-0.parquet | Bin 0 -> 457 bytes ...850273b-000b-4248-add7-b30a6c7bb5e1-0.blob | Bin 0 -> 100 bytes ...b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob | Bin 0 -> 93 bytes ...est-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 | Bin 0 -> 2333 bytes ...ist-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 | Bin 0 -> 799 bytes ...ist-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 | Bin 0 -> 899 bytes .../python_nested_blob_types/schema/schema-0 | 43 ++ .../python_nested_blob_types/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 15 + ...8aec-37d9-4505-a929-bd393a18b9f3-0.parquet | Bin 0 -> 1548 bytes ...est-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 | Bin 0 -> 2210 bytes ...ist-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0 | Bin 0 -> 799 bytes ...ist-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 | Bin 0 -> 897 bytes .../python_time_types/schema/schema-0 | 41 ++ .../python_time_types/snapshot/LATEST | 1 + .../python_time_types/snapshot/snapshot-1 | 14 + ...7d6e1a8-ff91-4765-b6a8-e6db2f3c008b-0.blob | Bin 0 -> 81 bytes ...037b-30b6-4b3f-89e2-d97896fb4019-0.parquet | Bin 0 -> 21841 bytes ...est-41694715-8b35-4d34-86c9-46164e966b5d-0 | Bin 0 -> 2547 bytes ...ist-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0 | Bin 0 -> 799 bytes ...ist-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 | Bin 0 -> 899 bytes .../python_types/schema/schema-0 | 387 ++++++++++++++ .../python_types/snapshot/LATEST | 1 + .../python_types/snapshot/snapshot-1 | 15 + ...5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet | Bin 0 -> 3469 bytes ...est-c830fb42-bbd0-416c-9349-d01b97497d7b-0 | Bin 0 -> 2205 bytes ...ist-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 | Bin 0 -> 799 bytes ...ist-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 | Bin 0 -> 897 bytes .../python_vector_types/schema/schema-0 | 91 ++++ .../python_vector_types/snapshot/LATEST | 1 + .../python_vector_types/snapshot/snapshot-1 | 14 + ...ab27-cf5e-48e5-8449-3f674480e077-0.parquet | Bin 0 -> 2015 bytes ...est-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 | Bin 0 -> 2432 bytes ...ist-6e560f60-1b27-45ff-b676-0bb081ff3261-0 | Bin 0 -> 1117 bytes ...ist-6e560f60-1b27-45ff-b676-0bb081ff3261-1 | Bin 0 -> 1219 bytes .../rust_nested_blob_types/schema/schema-0 | 1 + .../rust_nested_blob_types/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 1 + ...33f1-9c3f-4cbc-9a9b-dcc743380316-0.parquet | Bin 0 -> 2050 bytes ...est-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 | Bin 0 -> 2428 bytes ...ist-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 | Bin 0 -> 1117 bytes ...ist-31a697fa-9010-49d1-8d91-d1599ffbd01f-1 | Bin 0 -> 1220 bytes .../rust_time_types/schema/schema-0 | 1 + .../rust_time_types/snapshot/LATEST | 1 + .../rust_time_types/snapshot/snapshot-1 | 1 + ...12c1-f361-49d3-be0a-51dfc4777014-0.parquet | Bin 0 -> 27493 bytes ...da58573-1b72-4849-86fa-8caca9413789-0.blob | Bin 0 -> 81 bytes ...est-93f766c3-ee4f-4638-9cf3-823281ee15e4-0 | Bin 0 -> 2992 bytes ...ist-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0 | Bin 0 -> 1117 bytes ...ist-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1 | Bin 0 -> 1219 bytes .../rust_types/schema/schema-0 | 1 + .../rust_types/snapshot/LATEST | 1 + .../rust_types/snapshot/snapshot-1 | 1 + ...7437-0dec-4de2-be1a-44561281c6db-0.parquet | Bin 0 -> 4417 bytes ...est-e53c18e4-2a6c-479f-a696-fbb444f8df62-0 | Bin 0 -> 2421 bytes ...ist-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 | Bin 0 -> 1117 bytes ...ist-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1 | Bin 0 -> 1220 bytes .../rust_vector_types/schema/schema-0 | 1 + .../rust_vector_types/snapshot/LATEST | 1 + .../rust_vector_types/snapshot/snapshot-1 | 1 + 102 files changed, 1825 insertions(+), 9 deletions(-) create mode 100644 test/inte/paimon_read_compat_inte_test.cpp create mode 100644 test/test_data/parquet/append_types_compatibility.db/README.md create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/bucket-0/data-66214e22-ae63-42cf-8971-37326665ac74-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-fc7846ba-bc50-4067-86a4-d70bbc0d979f-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-1.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-92f63f4a-170d-4a74-a9e2-364bdb2c1e52-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_time_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-37d6e1a8-ff91-4765-b6a8-e6db2f3c008b-0.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-c830fb42-bbd0-416c-9349-d01b97497d7b-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_vector_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/bucket-0/data-08a933f1-9c3f-4cbc-9a9b-dcc743380316-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_time_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-5b3012c1-f361-49d3-be0a-51dfc4777014-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/bucket-0/data-ada58573-1b72-4849-86fa-8caca9413789-0.blob create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-93f766c3-ee4f-4638-9cf3-823281ee15e4-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/manifest/manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-e53c18e4-2a6c-479f-a696-fbb444f8df62-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index e2c8b4cfd..ca8a28be8 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -31,10 +31,12 @@ #include "arrow/scalar.h" #include "fmt/format.h" #include "paimon/common/data/binary_string.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" #include "paimon/core/utils/field_mapping.h" @@ -142,11 +144,22 @@ Result> FieldMappingReader::CastNonPartitionArrayI CastingUtils::Cast(dict_array, /*target_type=*/arrow::utf8(), arrow::compute::CastOptions::Safe(), arrow_pool_.get())); } - PAIMON_ASSIGN_OR_RAISE( - column, - non_partition_info_.cast_executors[i]->Cast( - single_column_array, non_partition_info_.non_partition_read_schema[i].Type(), - arrow_pool_.get())); + const DataField& read_field = non_partition_info_.non_partition_read_schema[i]; + // The Parquet reader may already have restored an inline BLOB through ARROW:schema. + // Only bypass the physical binary-to-BLOB cast when both the configured executor and + // the logical field metadata confirm that this is a BLOB conversion. + const bool is_already_logical_blob = + std::dynamic_pointer_cast( + non_partition_info_.cast_executors[i]) != nullptr && + single_column_array->type_id() == arrow::Type::LARGE_BINARY && + BlobUtils::IsBlobField(read_field.ArrowField()); + if (is_already_logical_blob) { + column = single_column_array; + } else { + PAIMON_ASSIGN_OR_RAISE( + column, non_partition_info_.cast_executors[i]->Cast( + single_column_array, read_field.Type(), arrow_pool_.get())); + } } else { // read and data type may both be string type, but after adapter transform, type may be // dictionary, need reconstruct struct type @@ -156,7 +169,8 @@ Result> FieldMappingReader::CastNonPartitionArrayI // read types differ -- the reader may hand back a dictionary-encoded array // for an unchanged type, which is not a reshape target. if (!non_partition_info_.non_partition_data_schema[i].Type()->Equals( - *non_partition_info_.non_partition_read_schema[i].Type())) { + *non_partition_info_.non_partition_read_schema[i].Type()) && + !column->type()->Equals(*non_partition_info_.non_partition_read_schema[i].Type())) { PAIMON_ASSIGN_OR_RAISE( column, NestedProjectionUtils::AlignArrayToReadType( column, non_partition_info_.non_partition_read_schema[i].Type(), diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 139b7fb20..3e73294ff 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -805,6 +805,52 @@ TEST_F(FieldMappingReaderTest, TestReadInlineBlobAsBinaryDataFile) { /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); } +TEST_F(FieldMappingReaderTest, TestReadInlineBlobAlreadyRestoredAsLargeBinary) { + // The data schema declares the inline BLOB's physical type as binary, while the mock reader + // simulates Parquet restoring the actual array to logical large_binary from ARROW:schema. + std::vector data_fields = { + DataField(0, arrow::field("descriptor", arrow::binary(), /*nullable=*/true)), + }; + std::shared_ptr data_schema = + DataField::ConvertDataFieldsToArrowSchema(data_fields); + + std::vector read_fields = { + DataField(0, BlobUtils::ToArrowField("descriptor", /*nullable=*/true)), + }; + std::shared_ptr read_schema = + DataField::ConvertDataFieldsToArrowSchema(read_fields); + std::shared_ptr restored_array = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("descriptor", arrow::large_binary())}), + R"([ + ["descriptor-1"], + [null], + ["descriptor-2"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, + /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr mapping, + mapping_builder->CreateFieldMapping(data_fields)); + std::unique_ptr mock = std::make_unique( + restored_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + FieldMappingReader::Create(read_schema->num_fields(), std::move(mock), + BinaryRow::EmptyRow(), std::move(mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, + GetArrowPool(pool_))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + + std::shared_ptr expected = + std::make_shared(arrow::ArrayVector({restored_array})); + ASSERT_TRUE(result->type()->Equals(expected->type())) + << result->type()->ToString() << " vs " << expected->type()->ToString(); + ASSERT_TRUE(result->Equals(expected)) << result->ToString() << " vs " << expected->ToString(); +} + TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { // Test all 4 combinations of rename × cast: // f0: no rename, no cast (utf8 → utf8, name unchanged) diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.cpp b/src/paimon/format/parquet/parquet_timestamp_converter.cpp index fc628e60f..01521762b 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.cpp +++ b/src/paimon/format/parquet/parquet_timestamp_converter.cpp @@ -24,12 +24,26 @@ #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" namespace paimon::parquet { +namespace { + +bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { + // ARROW:schema may restore an inline BLOB's physical binary column to its logical + // large_binary representation before this compatibility check runs. + return src_field->type()->id() == arrow::Type::LARGE_BINARY && + target_field->type()->id() == arrow::Type::BINARY && + BlobUtils::IsBlobMetadata(target_field->metadata()); +} + +} // namespace + Result> ParquetTimestampConverter::AdjustTimezone( const std::shared_ptr& src_data_type) { arrow::Type::type type = src_data_type->id(); @@ -91,9 +105,14 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( src_data_type->ToString(), target_data_type->ToString())); } for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { - PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedCastArrayForTimestamp( - src_struct_type->field(i)->type(), - target_struct_type->field(i)->type())); + const std::shared_ptr& src_field = src_struct_type->field(i); + const std::shared_ptr& target_field = target_struct_type->field(i); + if (IsAlreadyLogicalBlob(src_field, target_field)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + bool need_cast, + NeedCastArrayForTimestamp(src_field->type(), target_field->type())); if (need_cast) { return true; } diff --git a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp b/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp index 4d764e1ad..784eecd05 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp @@ -23,6 +23,7 @@ #include "arrow/api.h" #include "arrow/ipc/api.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/memory/memory_pool.h" @@ -90,6 +91,24 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { } } +TEST(ParquetTimestampConverterTest, TestAlreadyLogicalBlobCompatibility) { + std::shared_ptr restored_blob = arrow::field("blob", arrow::large_binary()); + std::shared_ptr expected_physical_blob = + BlobUtils::ToArrowField("blob")->WithType(arrow::binary()); + + ASSERT_OK_AND_ASSIGN(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + arrow::struct_({restored_blob}), + arrow::struct_({expected_physical_blob}))); + ASSERT_FALSE(need_cast); + + // Only a binary field carrying Paimon BLOB metadata is compatible with a large_binary field + // already restored from ARROW:schema. A plain binary field must still be rejected. + ASSERT_NOK_WITH_MSG(ParquetTimestampConverter::NeedCastArrayForTimestamp( + arrow::struct_({restored_blob}), + arrow::struct_({arrow::field("blob", arrow::binary())})), + "src type large_binary and target type binary mismatch"); +} + TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 75147ce60..271a52c62 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -78,6 +78,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(paimon_read_compat_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(scan_and_read_inte_test STATIC_LINK_LIBS paimon_shared diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp new file mode 100644 index 000000000..df6d86a52 --- /dev/null +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -0,0 +1,500 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/defs.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" + +namespace paimon::test { +namespace { + +const std::map kBlobDescriptorReadOptions = { + {Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, +}; +const char kDatabaseName[] = "append_types_compatibility"; + +std::vector WriterPrefixes() { + return {"python", "rust", "java"}; +} + +Result> ReadTable( + const std::string& table_name, const std::vector& field_names, + const std::map& options = {}) { + std::string table_path = GetDataDir() + "/parquet/" + kDatabaseName + ".db/" + table_name; + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + return ReadResultCollector::CollectResult(std::move(batch_reader)); +} + +std::shared_ptr GetOnlyStructChunk( + const std::shared_ptr& result) { + if (!result || result->num_chunks() != 1) { + return nullptr; + } + return std::dynamic_pointer_cast(result->chunk(0)); +} + +void AssertArrayEqualsJson(const std::shared_ptr& actual, const std::string& context, + const std::string& expected_json) { + ASSERT_TRUE(actual) << context; + arrow::Result> expected = + arrow::ipc::internal::json::ArrayFromJSON(actual->type(), expected_json); + ASSERT_TRUE(expected.ok()) << context << ": " << expected.status().ToString(); + ASSERT_TRUE(actual->Equals(*expected)) << context << "\nexpected: " << (*expected)->ToString() + << "\nactual: " << actual->ToString(); +} + +void AssertFieldEqualsJson(const std::shared_ptr& rows, + const std::string& field_name, + const std::shared_ptr& expected_type, + const std::string& expected_json) { + std::shared_ptr actual = rows->GetFieldByName(field_name); + ASSERT_TRUE(actual) << field_name; + ASSERT_TRUE(actual->type()->Equals(expected_type)) + << field_name << " expected type: " << expected_type->ToString() + << " actual type: " << actual->type()->ToString(); + AssertArrayEqualsJson(actual, field_name, expected_json); +} + +Result VariantJsonAt(const std::shared_ptr& variants, + int64_t index) { + if (variants->IsNull(index)) { + return std::string("null"); + } + std::shared_ptr values = + std::dynamic_pointer_cast(variants->GetFieldByName("value")); + std::shared_ptr metadata = + std::dynamic_pointer_cast(variants->GetFieldByName("metadata")); + if (!values || !metadata) { + return Status::Invalid("Expected unshredded VARIANT struct"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(values->GetView(index), metadata->GetView(index), GetDefaultPool())); + return variant->ToJson(); +} + +void AssertVariantJsonAt(const std::shared_ptr& variants, int64_t index, + const std::string& expected_json) { + ASSERT_OK_AND_ASSIGN(std::string actual_json, VariantJsonAt(variants, index)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(expected_json, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(std::string normalized_expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, normalized_expected_json); +} + +} // namespace + +class PaimonReadCompatInteTest : public ::testing::TestWithParam {}; + +TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { + const std::string& writer_prefix = GetParam(); + TimezoneGuard timezone_guard("Asia/Shanghai"); + + // Project every field from the main compatibility table in schema order. + std::vector fields = { + // Basic numeric types. + "id", + "f_boolean", + "f_tinyint", + "f_smallint", + "f_int", + "f_bigint", + "f_float", + "f_double", + // Character and binary string types. + "f_char", + "f_varchar", + "f_string", + "f_binary", + "f_varbinary", + "f_bytes", + // BLOB values stored as external or inline descriptors. + "f_blob", + "f_blob_descriptor", + // DECIMAL values across physical precision boundaries. + "f_decimal_1_0", + "f_decimal_9_2", + "f_decimal_18_2", + "f_decimal_19_2", + "f_decimal_38_18", + "f_decimal_38_38", + // Date and timestamp types at every supported precision. + "f_date", + "f_timestamp_0", + "f_timestamp_3", + "f_timestamp_6", + "f_timestamp_9", + "f_timestamp_ltz_0", + "f_timestamp_ltz_3", + "f_timestamp_ltz_6", + "f_timestamp_ltz_9", + // VARIANT and nested container types. + "f_variant", + "f_array_int", + "f_map_string_bigint", + "f_row", + "f_array_array_int", + "f_array_map", + "f_map_array", + "f_array_row", + "f_map_row", + "f_deep_row", + "f_array_variant", + "f_map_variant", + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadTable(writer_prefix + "_types", fields, kBlobDescriptorReadOptions)); + std::shared_ptr rows = GetOnlyStructChunk(result); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 3); + ASSERT_EQ(rows->num_fields(), static_cast(fields.size() + 1)); + + // Paimon row kind and basic numeric values. + AssertFieldEqualsJson(rows, "_VALUE_KIND", arrow::int8(), "[0, 0, 0]"); + AssertFieldEqualsJson(rows, "id", arrow::int32(), "[1, 2, 3]"); + AssertFieldEqualsJson(rows, "f_boolean", arrow::boolean(), "[true, null, null]"); + AssertFieldEqualsJson(rows, "f_tinyint", arrow::int8(), "[-8, null, null]"); + AssertFieldEqualsJson(rows, "f_smallint", arrow::int16(), "[1234, null, null]"); + AssertFieldEqualsJson(rows, "f_int", arrow::int32(), "[-123456, null, null]"); + AssertFieldEqualsJson(rows, "f_bigint", arrow::int64(), "[9223372036854770000, null, null]"); + AssertFieldEqualsJson(rows, "f_float", arrow::float32(), "[1.25, null, null]"); + AssertFieldEqualsJson(rows, "f_double", arrow::float64(), "[-12345.6789, null, null]"); + + // Character and binary strings, including empty and embedded-null values. + AssertFieldEqualsJson(rows, "f_char", arrow::utf8(), R"(["char", null, ""])"); + AssertFieldEqualsJson(rows, "f_varchar", arrow::utf8(), R"(["varchar-中文", null, ""])"); + AssertFieldEqualsJson(rows, "f_string", arrow::utf8(), R"(["pypaimon 2.0.0", null, ""])"); + AssertFieldEqualsJson(rows, "f_binary", arrow::binary(), R"(["12345678", null, "ABCDEFGH"])"); + AssertFieldEqualsJson(rows, "f_varbinary", arrow::binary(), + R"(["varbinary\u0000value", null, ""])"); + AssertFieldEqualsJson(rows, "f_bytes", arrow::binary(), R"(["bytes\u0000value", null, ""])"); + + // External BLOB descriptors, including a non-empty and an empty blob range. + std::shared_ptr blobs = + std::dynamic_pointer_cast(rows->GetFieldByName("f_blob")); + ASSERT_TRUE(blobs); + ASSERT_EQ(blobs->length(), 3); + ASSERT_TRUE(blobs->IsNull(1)); + std::string_view first = blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_descriptor, + BlobDescriptor::Deserialize(first.data(), first.size())); + ASSERT_EQ(first_descriptor->Version(), 2); + ASSERT_EQ(first_descriptor->Offset(), 4); + ASSERT_EQ(first_descriptor->Length(), 41); + ASSERT_EQ(first_descriptor->Uri().substr(first_descriptor->Uri().size() - 5), ".blob"); + std::string_view empty = blobs->GetView(2); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_descriptor, + BlobDescriptor::Deserialize(empty.data(), empty.size())); + ASSERT_EQ(empty_descriptor->Offset(), 61); + ASSERT_EQ(empty_descriptor->Length(), 0); + + // Resolve the external BLOB and validate its payload instead of its descriptor. + std::map blob_value_options = { + {Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "false"}, + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob_value_result, + ReadTable(writer_prefix + "_types", {"f_blob"}, blob_value_options)); + std::shared_ptr blob_value_rows = GetOnlyStructChunk(blob_value_result); + ASSERT_TRUE(blob_value_rows); + AssertFieldEqualsJson(blob_value_rows, "f_blob", arrow::large_binary(), + R"(["ordinary blob payload from pypaimon 2.0.0", null, ""])"); + + // Inline BLOB descriptors exercise compatible binary types restored from ARROW:schema. + std::shared_ptr inline_blobs = + std::dynamic_pointer_cast( + rows->GetFieldByName("f_blob_descriptor")); + ASSERT_TRUE(inline_blobs); + ASSERT_EQ(inline_blobs->length(), 3); + ASSERT_FALSE(inline_blobs->IsNull(0)); + ASSERT_TRUE(inline_blobs->IsNull(1)); + ASSERT_TRUE(inline_blobs->IsNull(2)); + std::string_view inline_blob = inline_blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr inline_descriptor, + BlobDescriptor::Deserialize(inline_blob.data(), inline_blob.size())); + ASSERT_EQ(inline_descriptor->Version(), 2); + ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); + ASSERT_EQ(inline_descriptor->Offset(), 7); + ASSERT_EQ(inline_descriptor->Length(), 11); + + // DECIMAL values around the int32, int64, and fixed-byte-array precision boundaries. + AssertFieldEqualsJson(rows, "f_decimal_1_0", arrow::decimal128(1, 0), R"(["9", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_9_2", arrow::decimal128(9, 2), + R"(["1234567.89", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_18_2", arrow::decimal128(18, 2), + R"(["1234567890123456.78", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_19_2", arrow::decimal128(19, 2), + R"(["12345678901234567.89", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_38_18", arrow::decimal128(38, 18), + R"(["12345678901234567890.123456789012345678", null, null])"); + AssertFieldEqualsJson(rows, "f_decimal_38_38", arrow::decimal128(38, 38), + R"(["0.12345678901234567890123456789012345678", null, null])"); + + // DATE, TIMESTAMP, and TIMESTAMP_LTZ values at second through nanosecond precision. + AssertFieldEqualsJson(rows, "f_date", arrow::date32(), "[19782, null, null]"); + AssertFieldEqualsJson(rows, "f_timestamp_0", arrow::timestamp(arrow::TimeUnit::SECOND), + R"(["2024-02-29 12:34:56", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_3", arrow::timestamp(arrow::TimeUnit::MILLI), + R"(["2024-02-29 12:34:56.123", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_6", arrow::timestamp(arrow::TimeUnit::MICRO), + R"(["2024-02-29 12:34:56.123456", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_9", arrow::timestamp(arrow::TimeUnit::NANO), + R"(["2024-02-29 12:34:56.123456000", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_0", + arrow::timestamp(arrow::TimeUnit::SECOND, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_3", + arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_6", + arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123456", null, null])"); + AssertFieldEqualsJson(rows, "f_timestamp_ltz_9", + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), + R"(["2024-02-29 12:34:56.123456000", null, null])"); + + // First-level ARRAY, MAP, and ROW values with null and empty containers. + AssertFieldEqualsJson(rows, "f_array_int", arrow::list(arrow::int32()), + "[[1, null, 3], null, []]"); + AssertFieldEqualsJson(rows, "f_map_string_bigint", arrow::map(arrow::utf8(), arrow::int64()), + R"([[ ["one", 1], ["null", null] ], null, []])"); + std::shared_ptr row_type = arrow::struct_({ + arrow::field("nested_int", arrow::int32()), + arrow::field("nested_string", arrow::utf8()), + arrow::field("nested_decimal", arrow::decimal128(19, 4)), + arrow::field("nested_timestamp", arrow::timestamp(arrow::TimeUnit::NANO)), + arrow::field("nested_timestamp_ltz", + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai")), + }); + AssertFieldEqualsJson( + rows, "f_row", row_type, + R"([[7, "nested", "123456789012345.6789", "2024-02-29 12:34:56.123456000", "2024-02-29 12:34:56.123456000"], null, null])"); + + // Nested ARRAY/MAP combinations and containers whose values are ROWs. + AssertFieldEqualsJson(rows, "f_array_array_int", arrow::list(arrow::list(arrow::int32())), + "[[[1, 2], null, []], null, []]"); + AssertFieldEqualsJson(rows, "f_array_map", + arrow::list(arrow::map(arrow::utf8(), arrow::int32())), + R"([[[["a", 1], ["b", null]], null, []], null, []])"); + AssertFieldEqualsJson(rows, "f_map_array", + arrow::map(arrow::utf8(), arrow::list(arrow::int32())), + R"([[ ["numbers", [1, null, 3]], ["empty", []] ], null, []])"); + std::shared_ptr array_row_type = arrow::list(arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::decimal128(9, 2)), + })); + AssertFieldEqualsJson(rows, "f_array_row", array_row_type, + R"([[ ["alice", "99.50"], null, ["bob", null] ], null, []])"); + std::shared_ptr map_row_type = arrow::map( + arrow::utf8(), + arrow::struct_({arrow::field("enabled", arrow::boolean()), + arrow::field("event_time", arrow::timestamp(arrow::TimeUnit::MICRO))})); + AssertFieldEqualsJson( + rows, "f_map_row", map_row_type, + R"([[ ["first", [true, "2024-02-29 12:34:56.123456"]], ["second", null] ], null, []])"); + + // Top-level VARIANT values are compared as normalized JSON. + const std::string object_variant_json = + R"({"name":"variant-object","count":42,"active":true,"items":[null,1,"x"],"nested":{"decimal":12.34}})"; + const std::string array_variant_json = R"([1,"two",false,{"k":"v"}])"; + std::shared_ptr variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_variant")); + ASSERT_TRUE(variants); + AssertVariantJsonAt(variants, 0, object_variant_json); + ASSERT_TRUE(variants->IsNull(1)); + ASSERT_TRUE(variants->IsNull(2)); + + // Deeply nested ROW -> ARRAY> and ROW -> MAP values. + std::shared_ptr deep_rows = + std::dynamic_pointer_cast(rows->GetFieldByName("f_deep_row")); + ASSERT_TRUE(deep_rows); + ASSERT_FALSE(deep_rows->IsNull(0)); + ASSERT_TRUE(deep_rows->IsNull(1)); + ASSERT_TRUE(deep_rows->IsNull(2)); + std::shared_ptr children = + std::dynamic_pointer_cast(deep_rows->GetFieldByName("children")); + ASSERT_TRUE(children); + ASSERT_EQ(children->value_length(0), 2); + std::shared_ptr child_rows = + std::dynamic_pointer_cast(children->values()); + ASSERT_TRUE(child_rows); + AssertArrayEqualsJson(child_rows->GetFieldByName("leaf_id"), "f_deep_row.children.leaf_id", + "[10, 11]"); + std::shared_ptr leaf_variants = + std::dynamic_pointer_cast(child_rows->GetFieldByName("leaf_variant")); + ASSERT_TRUE(leaf_variants); + AssertVariantJsonAt(leaf_variants, 0, array_variant_json); + ASSERT_TRUE(leaf_variants->IsNull(1)); + std::shared_ptr labels = + std::dynamic_pointer_cast(deep_rows->GetFieldByName("labels")); + ASSERT_TRUE(labels); + AssertArrayEqualsJson(labels->Slice(0, 1), "f_deep_row.labels", + R"([[ ["language", "python"], ["format", "parquet"] ]])"); + + // ARRAY values, including null elements and empty arrays. + std::shared_ptr array_variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_array_variant")); + ASSERT_TRUE(array_variants); + ASSERT_EQ(array_variants->value_length(0), 3); + ASSERT_TRUE(array_variants->IsNull(1)); + ASSERT_EQ(array_variants->value_length(2), 0); + std::shared_ptr array_variant_values = + std::dynamic_pointer_cast(array_variants->values()); + ASSERT_TRUE(array_variant_values); + AssertVariantJsonAt(array_variant_values, 0, object_variant_json); + ASSERT_TRUE(array_variant_values->IsNull(1)); + AssertVariantJsonAt(array_variant_values, 2, array_variant_json); + + // MAP values and their key/value alignment. + std::shared_ptr map_variants = + std::dynamic_pointer_cast(rows->GetFieldByName("f_map_variant")); + ASSERT_TRUE(map_variants); + ASSERT_EQ(map_variants->value_length(0), 2); + ASSERT_TRUE(map_variants->IsNull(1)); + ASSERT_EQ(map_variants->value_length(2), 0); + AssertArrayEqualsJson(map_variants->keys(), "f_map_variant.keys", R"(["object", "array"])"); + std::shared_ptr map_variant_values = + std::dynamic_pointer_cast(map_variants->items()); + ASSERT_TRUE(map_variant_values); + AssertVariantJsonAt(map_variant_values, 0, object_variant_json); + AssertVariantJsonAt(map_variant_values, 1, array_variant_json); + + // VECTOR values are stored separately because VECTOR and BLOB require incompatible + // table options. Validate every supported VECTOR element type here. + std::vector vector_fields = { + "id", "f_vector_boolean", "f_vector_tinyint", "f_vector_smallint", + "f_vector_int", "f_vector_bigint", "f_vector_float", "f_vector_double", + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_result, + ReadTable(writer_prefix + "_vector_types", vector_fields)); + std::shared_ptr vector_rows = GetOnlyStructChunk(vector_result); + ASSERT_TRUE(vector_rows); + ASSERT_EQ(vector_rows->length(), 2); + AssertFieldEqualsJson(vector_rows, "id", arrow::int32(), "[1, 2]"); + AssertFieldEqualsJson(vector_rows, "f_vector_boolean", + arrow::fixed_size_list(arrow::boolean(), 3), + "[[true, false, true], [false, true, false]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_tinyint", arrow::fixed_size_list(arrow::int8(), 3), + "[[-1, 0, 1], [2, 3, 4]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_smallint", + arrow::fixed_size_list(arrow::int16(), 3), + "[[-1000, 0, 1000], [2000, 3000, 4000]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_int", arrow::fixed_size_list(arrow::int32(), 3), + "[[-100000, 0, 100000], [200000, 300000, 400000]]"); + AssertFieldEqualsJson( + vector_rows, "f_vector_bigint", arrow::fixed_size_list(arrow::int64(), 3), + "[[-10000000000, 0, 10000000000], [20000000000, 30000000000, 40000000000]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_float", + arrow::fixed_size_list(arrow::float32(), 3), + "[[1.25, -2.5, 3.75], [4.25, 5.5, 6.75]]"); + AssertFieldEqualsJson(vector_rows, "f_vector_double", + arrow::fixed_size_list(arrow::float64(), 3), + "[[1.125, -2.25, 3.5], [4.125, 5.25, 6.5]]"); +} + +INSTANTIATE_TEST_SUITE_P(Writers, PaimonReadCompatInteTest, ::testing::ValuesIn(WriterPrefixes()), + [](const ::testing::TestParamInfo& info) { + return info.param; + }); + +struct UnsupportedReadCase { + std::string name; + std::string table_suffix; + std::string field_name; + std::string expected_error; +}; + +struct UnsupportedReadParam { + std::string writer_prefix; + UnsupportedReadCase read_case; +}; + +class PaimonUnsupportedTypeInteTest : public ::testing::TestWithParam {}; + +TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError) { + const UnsupportedReadParam& test_case = GetParam(); + ASSERT_NOK_WITH_MSG(ReadTable(test_case.writer_prefix + "_" + test_case.read_case.table_suffix, + {test_case.read_case.field_name}), + test_case.read_case.expected_error); +} + +std::vector UnsupportedReadParams() { + const std::vector read_cases = { + {"ArrayBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, + {"MapBlob", "nested_blob_types", "f_map_blob", "Blob field must be a top-level field"}, + {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, + {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, + {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, + {"TimePrecision9", "time_types", "f_time_9", "Unsupported type: TIME"}, + }; + std::vector result; + for (const std::string& writer_prefix : WriterPrefixes()) { + for (const UnsupportedReadCase& read_case : read_cases) { + result.push_back({writer_prefix, read_case}); + } + } + return result; +} + +INSTANTIATE_TEST_SUITE_P(UnsupportedTypes, PaimonUnsupportedTypeInteTest, + ::testing::ValuesIn(UnsupportedReadParams()), + [](const ::testing::TestParamInfo& info) { + return info.param.writer_prefix + info.param.read_case.name; + }); + +} // namespace paimon::test diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md b/test/test_data/parquet/append_types_compatibility.db/README.md new file mode 100644 index 000000000..db2486549 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/README.md @@ -0,0 +1,45 @@ + + +# Paimon writer compatibility fixtures + +The `append_types_compatibility.db` database contains equivalent append-only tables written by Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of types in that table; no individual table is claimed to contain every Paimon type. + +The writers and Parquet implementations are: + +- `python_*`: `pypaimon==2.0.0` with `pyarrow==19.0.1`. The Parquet files contain `ARROW:schema` metadata. +- `rust_*`: Paimon Rust 0.4.0 at commit `ced0c86b4db76265b5b2aedcbc9909a208f2c130`, using `parquet-rs version 58.4.0`. The Parquet files contain `ARROW:schema` metadata. +- `java_*`: the Java batch-write API from `paimon-bundle-2-ali-2.6.jar`, using `parquet-mr version 1.16.0`. The Parquet files do not contain `ARROW:schema` metadata. + +## Table groups + +Each writer prefix has four tables with the same schema and logical values: + +- `_types`: 43 columns and 3 rows covering integral and floating types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE, TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions, VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in every nullable column and row 3 exercises empty values. +- `_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, and DOUBLE. +- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ currently rejects these because it only permits BLOB as a top-level field. +- `_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++ currently rejects `TIME` while parsing the table schema. + +The tables are separated because Paimon C++ does not allow VECTOR in a data-evolution table, BLOB requires data evolution, and a schema-level incompatibility must not prevent compatible columns from being tested. + +`MULTISET` is absent because the Python batch writer cannot produce it through its PyArrow conversion and Paimon C++ rejects it while parsing the table schema. Paimon Rust 0.4.0 supports `MULTISET` and represents it as an Arrow map from each element to its `INT` count, but it cannot be included in an equivalent three-writer fixture. Consequently, this fixture is a compatibility matrix, not an assertion that every type supported by every Paimon implementation can be represented by one table. + +The `f_blob_descriptor` value intentionally points at a nonexistent external URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor and Parquet `ARROW:schema` path rather than external blob fetching. + +Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types differ in these fixtures. The Python fixture records the physical Parquet `binary` column as Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test verifies that Paimon C++ can read both representations and preserve the descriptor value. diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2d2cb3152d59b7a0c18bdcb044dcba1b6e638815 GIT binary patch literal 298 zcmX9)!AiqG5S>N|dr(2>ELqq?E)5Yh?k1A7(2Ex-o{Hc#nrt>0Fl|k(U*d;&@dNw; zKchb(f)jIi%)H0$do#1VzS9Wkp~A0^{m+R+mncI95Pn|%!r~EN1ON`E@=jV&#P0_UL)x*|2*|tE3)CqV_Q{4k_S4bfPko0l_O!P7yahKol zJYe&|b%@{){$~t~LTEa&f;qXok8sRujDY^R0pt0YhQ8et-OJ9tUbLP6X1i^(UimuG zX{7wH+%?aYf2CrROtmse7FY2sRf*M8on^W(wydo&nJ#Ue>N6oFZSzf7(v|)Ib$BuC literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob new file mode 100644 index 0000000000000000000000000000000000000000..6bf912ee260bcb6da88a90c5747ed5cd57cd5338 GIT binary patch literal 93 zcmX>v=ojJS?BvME%)r2ql#`#Nn^;toSgC7(#5R(Ulmbax05J$yu9jJM9;Oc>#K0g6 QWz4=ZF}$GA2Be-501j#seEv=ojJS?BvVH#K6FilbTjil$oAUl9ZF5q??;qpbHY=V3Lvs$%3#o5QBi+Q_n-^ XVOk+V5K~_=(&@#Mp!-= z4p|kl4lcwAti;LX?K~Ioi=eWlFTuprL&}Z2NOTx~7ax*9v`h!o;G`qSREa1U}TUu4x73 z#+wbOKsX6Qxf&R|%R?GwGhv)K$<)9mt#M%1$B8v=20;vro5pBeo-GN$$h=zN23pA7 z)nkMQ>clA;+#E%E2tt)ZCu;$r?mz$_KgI3E3If}U#NZ_^C_6EJ4iw$IQEwUWbM0(* zKmwI=Wz-urno(&&n@6S8s5efD>I6oeQY-!crI2sV4eLCRv?DiroHj`59y7ZTS#0ZQ zx4jxVF$OmT-^573dPwMROkNwXk)|*ugt{5Dn@Tc)7C+-v+bM|2i-n})qN@PK~9v|V5B_A2Le3WMkZewHk<rV z49uX72^%$OG!`}Z={iw&j>|*Y^{u0o%L=C}He>^(kkDmMFld^kl0O|gx=%=vXie8* zbhUZ5_ySGGP=!-&T2a~Ht3+>`OWZ)==#u{ZU^=fBkiU>ljfH7!w6(m^y~_937xr_$ z+_T7zAwMhlEibb6^v|8h?i#ASyx4T^%5c`N%#!gVtCs4~9hP_5pd>c_IwLxIe`4ru z>)2e+)WW^@KWA5;!P_^6yJnVOdT(WYFZwpq21-|K^V8LbBLz91(8JYJH>W4ge_q=+ zWt+F|94J5ZYWd^y=*gHRHZ>Xiy>j8=;NXu-%S%g3p4wd_14V#;axAZSENAw3owcCv zVX1g0di_kn;xIcz6n2g5G+VV2pWLXq_)n+G*CZN zG1O@BJ!0CJMoG%Gx!90#Ca7eJK8RQxW}w-deJhbJ?6}CWi_L{s#3jfZCEyI1kl4Bf zOGqX~#D{g1i*72YI9TZmG8xX+RU=4X~Gb2P0fWATHK%srpBGgw%;^ z>ambby1MCAkK3P0miDL>eY|k`F+wttmDSLoV;usWtFY2d4&}mUkBw3;+wb%#q zj{MwjfRGL7B$lA5n7N->3_{Djc! G!NVsw^+cKg literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 new file mode 100644 index 0000000000000000000000000000000000000000..071447210821caf9daa6f0b6758a08ce1b97e5e7 GIT binary patch literal 1111 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z2Azkx{RwQW1{dR=`6Xvo++$k9sIgW5FM~%e6GPJx0m%tlt^QAs2zZ*jyZ6_8hirWo zzTfw(r93TJkCnz=FMU049-C7_#k!>&*Dl${!3|k*oS^BwIlUMVv897_3W6dchU9X=1LH^hm>hI- zbaZrfY^jrUD9)vHFBG~IK?K*T_`JE0XyHH#!reWe{NDfjb>+9S5&>VMjfdA?zYm(I zaj2sXV1Isd2Q$+E2>=lMvgKMDNAUXU*ZZh5)$1%rUCG4$@F&sh{dn;Er-*O<`_j$E zkT5cungbg(4HO1(&`v9gNz94b4qtkWb{U5J8T9OC--Vxn0rH}$)o&JbGfF) z)I6@GVrnI>>Af0hhG{HWV6w6t=!TMk1EkRu&0G3pw3yM#64>pfjUt{kpA*&#lK^zu zhiSSokyEEO$*jC6V*OMUlR7C=qd4wBHqLi-8S(ZEm~&@MyfrGz6#lk zo+5wrA<>BzBhu?4kHoW)e-!dt9J16!UOpdruaLJ2Nf&}NOQ-W>nlI{yjcRLiC)gfs zwz9iHYb#t|%gVg;@=n&S?S!lK?3h#V-3jUVZn01=lsv}`Ia*k~YbjwVmL_Tifq>pYMI2?|I%v zhc9?*_pldLC6G;x@)EnTH-~8B86F8DDxXqR4)G&0E4jXBGi*x#QBl(Y7JeS(0~uD3 z6ggm!rFofGS&i4rM+9EvvPdwID>5;0KvGzgfQN;<1X)A!ejCt-5#jQu{U}Le^cI5Q zagEPzq3DW*`T$*?gpSfE&X_2wJgghTfpuC%8-^*Mw;5SKVF)y0lOb7673B1mrVSK@ zopYc>r36Dp2bfSaMmT13VImm8^ntThn=Ai2))6-5iT_0n+xE+lA0#J4aKY zUJyYU%n^JSBM#yKw{I+68(@$oQ6q%58LXEoF@Y67=TnDBfa;5dp)p5Ni+-`FjK2KP zK4~GvinO^eH9_LD8Y^WENrET}a2vUN<=AjxfCCyZ?l2cGE10v?nhpv8MvG5ARTelVNY3dmnbXU4)cHr!gte3SA2n^HJFwfi!( zqs*}?=ed--^Rj*7!`%GMsgLib*CuQ3)wDf3yzc4UU+d|5)3R^!;L_DAFP85wEKMBf zkgwlfo)&*^`qt)W8e12>EWQpsZBNu~E+|d6I@@M8)<1XVR=;yAvn?wtyOW&#w=5^!ZAh;7}1 zB_tO-W}`aFHSa5^B-_B)zJ+zefCg5YZmG8wX+Q+M9k4fg*E2mC0lS{=m8yS~hjE>_ zr5?+)aZg8n^|*tXp!tvVmVUPb%Jvb$ZO6AOr|bhZO%fpcUe!-HCo@@1ENH4Gwb}>E zj)Ed+f`|^uG!dYsm_<-jhvl}fp`~Nf_@!JG>C$g^ptlCCUDWVv7M07Og6_?gz6kap z+Rt`hs&wr-x7AA4X}>z%Y;+^;|Hwgcx_otE7SZtJDHnuAghvhM+~^#=M2p48>s#X@ GeEtM2cSH67 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/manifest/manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1 new file mode 100644 index 0000000000000000000000000000000000000000..c0a0585fe524cbad8a58f8cd80d17aae02383c3c GIT binary patch literal 1108 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z2Iio4ebtZ+K`Wx}iw5y31~DyT)Yz*3m%$^Hi6MzaKw`mG%m4ouYld0fyvMSHt75Xi z^D3Rv?3K%MW3QLKo;EFt<4*LUBYCNEo1{xGe4O)>S9|>@iI&x}6GYBBTybP!WMVk* Kz?eY<-4*~4uWS?m literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 new file mode 100644 index 000000000..02f0fbffb --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/schema/schema-0 @@ -0,0 +1,34 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_time_0", + "type" : "TIME(0)" + }, { + "id" : 2, + "name" : "f_time_3", + "type" : "TIME(3)" + }, { + "id" : 3, + "name" : "f_time_6", + "type" : "TIME(6)" + }, { + "id" : 4, + "name" : "f_time_9", + "type" : "TIME(9)" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788853436258 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 new file mode 100644 index 000000000..651a6a907 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_time_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40ea5cbe-9f6a-463c-a506-4b8a21041b6d", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-277c98c6-c631-4fe8-9bb8-4f3ca1edb0e6-1", + "deltaManifestListSize" : 1108, + "commitUser" : "3adee06a-0e07-431a-aaa3-762caba0f779", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436279, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_types/bucket-0/data-f9e2c4dd-e99e-4a94-831e-a2261e62fbbc-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..aa712022fbeb8a49f62553cd2df36dd1c3125d78 GIT binary patch literal 16680 zcmdTsdwf(?diUN;CPNrPc%0)58)Kwo3|RtskPvVd5dkA8A|@=d{D?Ejgd|QT*NgzT0 zFhcG<=YHpVf8Y7e0~cJsFpnt2LvqQ!_7h*Hq!A}HUBsm*S-+q3jZ$)_qNs|ZXz(`y z{=z$aPa!_CuXEc&!#TLq052Axs>&fplL_R(uRnZLwLvbBkY_GjxL^Rk|Ju&2{Qw_! zNFEULXCG{PxgX%YSDqJ`)5v7<)rFoHhuYX^kZdgY82Gqw;THxgM~=OGTqwjOq0YCW zv|h)75qrnJV{at)jr||bmy9G{QqDy_hT@ zk38GbWf%Ck9FzjUcLy4AT+Ro(cb?evfOUA!tE;yTv(dB+Z&=k3sHv-uWaLlGotSG3 z_mfhx`NW@|u`8roQU{+=^70EN7Zyz^#`o)I-Y{$SoEuBH%+tt1()r$oPwYPPkwh>Y z1_ArQnn<8=6}*L4gbW7q$y{>!zwZA~v+>_4#}5VQ;Hqe-DSqrLD%r1+GjFc>Ceb$f zWlh=e#;7r|c}v_oMyz_gOu`1+PLY;Bs%wgbnxdh~%1}j3T_DVDluc%mw-3Cx8D@cGm?NFUDb4we zFsBn|(1dF-8cmZa=Joskvh6-d%r5J=<6@k~#?578>LVeX8X_ygCbH|F-TWnxjT z^5OH)>(~!xmy(I(rN8~^tUc8%=G-M{N)o_x=&@uP`TW_>zqIGi2X$zWSK)f$V=_oF zdFboc%WP_n=|e2(W5tY>=x_T zU-Pki)J(F3{N1U8A)Ax9gv(1GLz72V)CEJ0O^NUts%wa@;wVtuhqpc6Z6_?nt>)r} z9$?}c;3f$5NF|pT=gYr8_}HhwcN~h9 zGBP>enj&UcoG%|au@8ePZiJjKH95jOZ73jLDLK5Y^$9z-Qbl}Vj-n(3m-vI#4*qan z=aVY3cjuN`8{uLpEhC(X^*i{9L(!Z`DQm%s34S~j=t_TS%Q1Km7$*b>oJqojHoYTCcDLrqHt7G#Hs|C3U z1-ai$d9M1}SV`90d+z(EgU)@Q|Kf$m!ejquzWT?{*Q+1yay;dXb=-F0^(Wk;J}w9z zI`ZQ`%pUstu_Kc^0?*AY_}%13ZusTV8Ck7&`Ogpg>(lk0{85^%7fL}i@P z_!m1Iyu*QuQh>vUxf`np=*EY+8yA|;jSo||OD-wB9{tty8E~{-2cz&|>UPPcB8-x+ zrq4(o*LT>Zab}?_#k%T?`)FO$nLfJYat&z7)tc!uARZpK91-s1WK2cHBdH1qLU;?Ie~CEC%UR3)RYri84Wc?0^yusxZdlz zToet+jx;8&uRv}1Fn8lh3cB%O>UPOB6X-;~nm!NFOBS!kcsBv0#xIDx9$VMo!5z5x z0jBX`!s3$49AH|$4&bw#i(s^T9l&Twun*c2WADfd2UW>?J%bx(k}CVs>wB+f0Kg^^ z@!7`d^*J5hIhqD@IZsDe#(O;p5E*P3eAm21`e&%k_z$Q~z9Jmr(HI_)0US90fmh&z zAsSx<6FjHkTZ1g-Icus2qWS%Vvd5fH=mIEwT7ULa*X2avL)GpJR;^q@$t zNv~y_*VB8ylpL&tNOUE$ip}duWY0c;oV+3fmSmY`urvp-38N;7nhVs#>o5aq5z-W3 zgNxOR$z_QoF3o@gm3xLMA`@9%1oH4o$&jZ=_pWbW8i$N$sjxvxW5JU~U1S*)IEpuo zR*_QiwnrsP#Op>)e?4Wqb|#RZ0BCdQ0IL)A-V=$W762dxAm%P7xtF!luyrWO+&btY z%_{qt!9EUwjThETXU$}?lC>zd_75dz_{TOC+pi4MX>Uo*f89P(ANJKqeb~`a`s<$2`fCp%iv)&Fs`s{Cp}(&4=&v1K9d_#&*j8pT)FL0p ze?Z!KvlN>xSCR&SG?p&(l7b)3r|Tb1T}74(m;)K4n&EOI%&x0-@lK87jFF&_y_l-H0ooFNZ>>B`cFbc~m#NyRVwvl(rCeg21X471A%6Ldbd zG#6W;o2MDj5U1yYCByj9MR`+2b*L^tN+GppDjuz-vJA4klBs!yRFwzo z>%*Zyq)fw)tS4F%SydB>W~LW9(w-aT8UaqlJ|$O`H^Js!7{6r{I^kQY{w=Yp9J$`m+S1KWO3El7Ka`CZeIDKioUj_bYz%LYfOfb(SY8=w zs%Wffh}JijrE_tIRY51)&MeO>&&|X&t41=?mf};Y!~`?7JRe{jWF#32Fz`eeR7Q#k zPq6?RN5(OraR`(GW2|5jmyh@u7);6tmH}YH4QvI)<$1+Ga~!#uWC0vJ`+_S#xSQd> zY=DC&!lAW6?UjLODAQdyAuWuZ$S?>Mt*L|k+(2Ce2tKlKV%j>c{oViy03f$NfTGON zh4~0D0SEvFXFQ3iWs^B!u!TW59Q_$1UX-?O1hcUqZUaIFiZ2F)OiV){6B9N}Q_K8H zp>#XkKPQga)6q>7Pa->f!?3}Yt1c7`fZ_rWH824DjLni50^lRg!Wn6PkhIX@afD%g z2kyd@ofHa(>R_=~7K^QpIdI^ps`9!(gK%aMGJrrUoXPkE1Vio$t%4wcgFkR72dAC@ zec<`$KKuGY0{4Ejn&2?jB5*@QHrC%ml4sB2)RTGH%ffUH_x)j_OAn?Jj!rnF_$q<` z9LNQy+;hQ!;3*B6Du8smmEFuOJUaeEIT*BTK1vOG=jyU5Z9%-!092}sX*CwRKjw%B zvTC*&K`bT#{j$Le;|E%?YlX2E#E$OKzuM1!c2=q#BS@&d8Al7kXi_E`GiB)|;5_$d zi4(Zu7`eo)7nc}8EE1#D(FqszC$VF8tQ`lkJnO&%9q_2d-26U{p$L<^`XX}gU4wFC$RxqU2lG~CStVj|vz9nHgUGMq(bjL| zJHvrsDBQ#hU3)2-`0f8m=fMe%;|9&gx_~LBeP9GqJdX^?3tBUL5vh(dsVDUeO+u5vP(9=VoGR1lToEmWL<%P3$NN_;@selnJ!(F`=A>S1VR(j~q%Nt~u~Nk_U-1EjcF z%6}tLqh&?fjYtp2Bjt6Lj_(xW!(CSX8}S(}E50>%0>0Rf1d|RSxL&C-lyhm&*O}lq zc4v(@K&|iD8ogE$+mXccKO)naZe4n!^RsjZSS>`=tL4*rwgXtf`q6?8G(hg)^{S&U!-NRjb;z7v7}?dqEDg6 z?j&jS=yaGB(KRIsqk3gns_4DGE+=A%+ncuHgW{K-MSSO{lW7)wdey6=JCwxG$-QHK zL&=VAQvlHIzRr%0SH=uoEp1kP2VICy-}v_EyWwVmGcXndB%StL7pF6Gxy_JP)0pZz zGz5{xZ<|}WOb47@h_ihL@mqwJ zGHaSvRIut|7Q$@&$}q?ocwWJOW6m&ICftCyEeHYq$_;VZ*n}5WS32G;DS#IbJos?9I9I*ZI?R*=NvSD8$#CA87EJJoMLy|MQIZQB`-9m>i{ za|Mb$n=I>xlY(NMleQ7x87EtilQWVN>6v1!th>(kk>EOb?8JYg>liJoXwM_Ux8}sn zz1DTM?m1rpUj59&0@==_{Mr}}z`Z#zlzxXl*}tu3J0&sW7e z0TYSXrBz>^d!o#>!7x*f>SSEQtU(lV^qcEzA+?Lu1TE=tR= z;5yHsE!c^)_S_U_L7NUuR_yTlPIdt5~)= z`&hP)H;%Uz+8%;XXun0wv~EtTw;e!Z&kuL{K<6%si-%aczsQk=N8=s`9+R7eNihzE z6}mqJYtDv1TH4u`E&?)dBASGQD;^!y(T_DUU>|!(D#|V^T|S?Tee`& z;@gs@%}C}`v*=RasjEP#SUvOK;030ZiO5uTl&{b;h`;$(Qf%>Bj^Dbo$w(}q)xHB6 zetGIIpFl?31CTu?Qq;{){1y6e73g%&tz-rxCdwosOp0}*lu^UVY}eI;ugs*7ZB-!T z*;`@99+#G41{8E{d#{Z0y^zLyC&&6Wjr~@WsAOm_Fw_e#y>9m!{A?ZP>-ekhgrB!0 zKRkFB+c%djA#7;?e{Q;zMtwcw!1eG@N5r!$%hWRY!+6Ub4~3pY>+D}ZN`wnyq%GFd z2W+%i$l2oBO>sqgBGX^w%2IM`f87nVatZdoYE7*h5^h6`rVux`+#0v-IY=3VU+TyB+qkn ziLPYW^R|#<`{qD1nX>JZj`98?XBOVE@$d^bCPFk7q~sTB6tw0?C}->KWES%rE@$6_ z64`D%ql9T}e>O;mH+A@L3ep)Z6H!RV`W+sHuBisuK9D5E{A29q_hrB5e~9)Iw|+c{ z>=uxT2Bl{6ZLbEgo+%}Bm{?6EKV`U4p1O-{pw-~e@Cug z-1ia>V42@B5^Xa}$#E=guYpYb_K(S2lZs1BDi()Ay#!R0u2X#*@-U$Edt`71&o29$ zDW)-s{9c(tch`Ulo?J}kUn)J50U5;|`7DI_jWmzEqh=;Jh7lB3)D-#_QtXMN7=J8f z@n@Z6GKCG*nC5FMz+~8m8w_r_V`g}x3LMJ|Z?J@^fz8CdcgR9sB<_AWHakT!-J{aO zs;{jOnI6dEgFB6xvt<^>Hr9eB*ZhREnBBJ+b}&q*aJ@JRn>o)FF|+ZDcQB>kKTt4J zvSi*}NNCeik$Hsz`l7?FrQ6IUY}Qbymya()qtTO-sMF`72d^Cs|9Ek(xZasl*O+lP{DZI>_-EmH6Z48D=4NCC zSHKx`#^l`mih|<2+=_xJ`IY%oi*gG>dBu5CrsM@GLcyv~Ma7i7V5q7nZ#*1{!coun P?vHW@lpPD;|B>+j)v=ogV+l#-d3SX8Nyl#`#NP>@)elb@KPkXDqRt58r`keHd9pQm7?XP{?b$p8U| Y=H8b)57a0C@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~t}oF( z+|TVB!*|8oCL_x_!#=JaySy|?FOr||6VV-`3w{oA*m}Tc^+~&GYPuxJ)e^%$$ z+G)X8eUBDi2{8{z_o_5WT{SP`tmhhKEYs$j`r&M1M{c;QU8#x z_}{r+?3cdK(Lcwx9A#oy6?pMw!kQCHRv0wQGD@4{aq|azy9}@O;-E&qMlDCKqq+5M zo2)w0uV%7;T(jv>YUqZw9BRdthNnXh{yXs@JG$s9&+l7Zx0((Z$iBZ`cEu=ePHe7w zuIG$x5xLe`KXy(?xE;7GCF6G2r;6xz*UP3HzdcR4-<)UVi>%iZk8?}s=XpzI@_K&C z_GWs2Jf-g6W>w?M58aOZv43v=+1TDU?Z10l{OvwV`}$ckC;m9`!8%u`-t*`-B^7;@ zcbmdCtUb=%>8Y>M7TBZd;d%5DcjxV*DIq!C3jNy?MUrocsq}wI`jzFq%D}$fzx9%p zGoSZ7uBD}#m)@*;bT-#DYgxkWKEA8DcNco>__iyzCENe?T4yQW6&D@U3eNO+%;}l1 zYy$HICQ0qJd_oF>4L=OJ8yC1Ru^ja^ImlwXGxF9nW(6OP)U+cNJ}?e{^|w& b`^p1OSlld%7S6LXJHfvsNxp6-dddd?nuGV3I3(Kjcx6fo6C1twcMhZDY$UHkV#uB0=6L0T-Ac zYC;2*kW7j&8PriWzN?_3bPZ#=28FtgbgDG%Qg0(tM+7-rV6S1@!A=o@+75I|)j!Ik zv`*YmkAY3vfyAvIw>OnM`;p#&w_Bi0mk`>rT(fe@E=a~v1d6_w^)n&Zw5TSOJXVuh z?t?`~UhXwO!24_*Nzhcx+{>%OvRjwPfE4S$l&d10yR9qatUxOlHT;@I`x$X@fx} JlV|)Ed;&kLMJoUR literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 b/test/test_data/parquet/append_types_compatibility.db/java_types/manifest/manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1 new file mode 100644 index 0000000000000000000000000000000000000000..f62987afd10dce6e38dfa3d267427972d6cffb03 GIT binary patch literal 1115 zcmbVL-%Hy-9FHZ>K8!NM=AEbrjtx9=A7SB>f}3q3k^aWjeZ!nk~mHobmv~ zM_~wf-{bl*rFh7z2^fj;NsafxsH1MytpJbs_$Xwcku$SyRvebq+=hmNH1#j#=8@hz z%^he@KoggI_;VJ8k0B4bFqix4umknQY!{_+*LG{SSji;quTEzhor(K@Ipy2zNgpa%iF8Zdk^~SbGh_&>CLxy55o4Bl5io4A0r$GK1rFhS@qXz zejQ7z8!Mhv#~XDCZ<4B1Td()LUe#SCYgK9G?yV@i;;SU^YqG?W@bji7lxNKz0Tc*h AJ^%m! literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 new file mode 100644 index 000000000..554c947a9 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/schema/schema-0 @@ -0,0 +1,306 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_boolean", + "type" : "BOOLEAN" + }, { + "id" : 2, + "name" : "f_tinyint", + "type" : "TINYINT" + }, { + "id" : 3, + "name" : "f_smallint", + "type" : "SMALLINT" + }, { + "id" : 4, + "name" : "f_int", + "type" : "INT" + }, { + "id" : 5, + "name" : "f_bigint", + "type" : "BIGINT" + }, { + "id" : 6, + "name" : "f_float", + "type" : "FLOAT" + }, { + "id" : 7, + "name" : "f_double", + "type" : "DOUBLE" + }, { + "id" : 8, + "name" : "f_char", + "type" : "CHAR(8)" + }, { + "id" : 9, + "name" : "f_varchar", + "type" : "VARCHAR(64)" + }, { + "id" : 10, + "name" : "f_string", + "type" : "STRING" + }, { + "id" : 11, + "name" : "f_binary", + "type" : "BINARY(8)" + }, { + "id" : 12, + "name" : "f_varbinary", + "type" : "VARBINARY(64)" + }, { + "id" : 13, + "name" : "f_bytes", + "type" : "BYTES" + }, { + "id" : 14, + "name" : "f_blob", + "type" : "BLOB" + }, { + "id" : 15, + "name" : "f_blob_descriptor", + "type" : "BLOB" + }, { + "id" : 16, + "name" : "f_decimal_1_0", + "type" : "DECIMAL(1, 0)" + }, { + "id" : 17, + "name" : "f_decimal_9_2", + "type" : "DECIMAL(9, 2)" + }, { + "id" : 18, + "name" : "f_decimal_18_2", + "type" : "DECIMAL(18, 2)" + }, { + "id" : 19, + "name" : "f_decimal_19_2", + "type" : "DECIMAL(19, 2)" + }, { + "id" : 20, + "name" : "f_decimal_38_18", + "type" : "DECIMAL(38, 18)" + }, { + "id" : 21, + "name" : "f_decimal_38_38", + "type" : "DECIMAL(38, 38)" + }, { + "id" : 22, + "name" : "f_date", + "type" : "DATE" + }, { + "id" : 23, + "name" : "f_timestamp_0", + "type" : "TIMESTAMP(0)" + }, { + "id" : 24, + "name" : "f_timestamp_3", + "type" : "TIMESTAMP(3)" + }, { + "id" : 25, + "name" : "f_timestamp_6", + "type" : "TIMESTAMP(6)" + }, { + "id" : 26, + "name" : "f_timestamp_9", + "type" : "TIMESTAMP(9)" + }, { + "id" : 27, + "name" : "f_timestamp_ltz_0", + "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE" + }, { + "id" : 28, + "name" : "f_timestamp_ltz_3", + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, { + "id" : 29, + "name" : "f_timestamp_ltz_6", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + }, { + "id" : 30, + "name" : "f_timestamp_ltz_9", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + }, { + "id" : 31, + "name" : "f_variant", + "type" : "VARIANT" + }, { + "id" : 32, + "name" : "f_array_int", + "type" : { + "type" : "ARRAY", + "element" : "INT" + } + }, { + "id" : 33, + "name" : "f_map_string_bigint", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "BIGINT" + } + }, { + "id" : 34, + "name" : "f_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 35, + "name" : "nested_int", + "type" : "INT" + }, { + "id" : 36, + "name" : "nested_string", + "type" : "STRING" + }, { + "id" : 37, + "name" : "nested_decimal", + "type" : "DECIMAL(19, 4)" + }, { + "id" : 38, + "name" : "nested_timestamp", + "type" : "TIMESTAMP(9)" + }, { + "id" : 39, + "name" : "nested_timestamp_ltz", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + } ] + } + }, { + "id" : 40, + "name" : "f_array_array_int", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ARRAY", + "element" : "INT" + } + } + }, { + "id" : 41, + "name" : "f_array_map", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "INT" + } + } + }, { + "id" : 42, + "name" : "f_map_array", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : { + "type" : "ARRAY", + "element" : "INT" + } + } + }, { + "id" : 43, + "name" : "f_array_row", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ROW", + "fields" : [ { + "id" : 44, + "name" : "name", + "type" : "STRING" + }, { + "id" : 45, + "name" : "score", + "type" : "DECIMAL(9, 2)" + } ] + } + } + }, { + "id" : 46, + "name" : "f_map_row", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : { + "type" : "ROW", + "fields" : [ { + "id" : 47, + "name" : "enabled", + "type" : "BOOLEAN" + }, { + "id" : 48, + "name" : "event_time", + "type" : "TIMESTAMP(6)" + } ] + } + } + }, { + "id" : 49, + "name" : "f_deep_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 52, + "name" : "children", + "type" : { + "type" : "ARRAY", + "element" : { + "type" : "ROW", + "fields" : [ { + "id" : 50, + "name" : "leaf_id", + "type" : "BIGINT" + }, { + "id" : 51, + "name" : "leaf_variant", + "type" : "VARIANT" + } ] + } + } + }, { + "id" : 53, + "name" : "labels", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "STRING" + } + } ] + } + }, { + "id" : 54, + "name" : "f_array_variant", + "type" : { + "type" : "ARRAY", + "element" : "VARIANT" + } + }, { + "id" : 55, + "name" : "f_map_variant", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "VARIANT" + } + } ], + "highestFieldId" : 55, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "blob-field" : "f_blob,f_blob_descriptor", + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "avro", + "file.format" : "parquet", + "blob-descriptor-field" : "f_blob_descriptor" + }, + "comment" : "All Paimon data types written by Java Paimon", + "timeMillis" : 1788853434699 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 new file mode 100644 index 000000000..6483d0d61 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "b90d1c93-48c8-4f97-9201-484b9c9192d3", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-caac218e-853c-4ee6-94a3-2a2b509bfdee-1", + "deltaManifestListSize" : 1115, + "commitUser" : "2ec0ad8a-b06e-46ff-ac6c-270da2027e88", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436130, + "totalRecordCount" : 6, + "deltaRecordCount" : 6, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/bucket-0/data-325a3f55-53ea-4a75-81e0-0af91acbd76e-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c850af24c025cde8c1cb690479d8ea7dd21bfc7c GIT binary patch literal 2416 zcmai0O=ufO6n?v!N^5Cl%fvI;HA@i`3;9sTvLo36)2Ka^;FAe;j&UV<-3s*M*s^It zDQcj_hhRkNL)(*SFD6Zkq11$)gnQ}9F@eyYLI|aV9FkH3X^WHez5UTzUMXI*+L_t+ zec$_;H`2`8=TjVbj3@cplimHO!5K~!t^g!|I`tgRWdS4rknzSRzBBk~{@cN`Cn1BY zI1TckYr1Hg1u_9N4!rE+Hej0P`40~^hkf=*$)au9zr+iXEj^-P_8f(X=G*T-+X)}b zqaD~h50PL$1aKFsKg(ojhFO*CoF8 z>9ZYwA|4(I42-AWJ$Hhm0O60wdz4W4_2d=>2sht2z$^kZPR;b-!^_*TEzI zK?ABnM+3!8wXP{{Hai-k5Ah~4QJOj$U<1%4qmeTl^cxyN;TJG+&<$~s=bceg;_H$# z-jjL59p4Lh`DZ>fq$L=whx|u2$xj3m=;09FJktL!imU`B{Lp zti=}QR?R}Kx-^%sR!e5C!et&z%f=MClx7+$6_;yV#;zTbwUAjd%VwoETW{3stQ}n| zRxTGSXo_g&L(7NIL$9ann}|1Vo>2NpMb`RTPAr#mrIO1Nd^JxzUEj8OVh)em;&CaI zS5nN@n@%8oEo8pQv-N!k+}i>#l&U#92dWRNv|;c2gy*X(`I2cRu-69; zw}Jm6uwc5OUrT_2bshpmrBKBt8^se@z-k%XC zzu}`1+wJ2Q+>*o%+(aRY*t!O@?nox(4y)Zf_?kS}QZZt2DJh!G9#n$WPYVsnvb6z9V?7arbY|n6JzsZXELMXW@<7uF_Fp@ e%>05`C`_dC=0YYltY0$#I)32xYXCF&U+aIzdEr$6 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-30233ec1-2b71-4793-8005-99a4aa0df863-0 new file mode 100644 index 0000000000000000000000000000000000000000..85f300079382d3dff3e739d89b92e99b0c4799eb GIT binary patch literal 2189 zcmdT_O=#0#818;VDjxh9LpbS(9}cX$92KrP%0wJJDnY<=~08oXT=XEMin8Qz$VPxVsVD&ZsbVid(}XE3hOA3D!-hr_0< zD7>{VbvY*$bs<+coRbwr#@oo_E7yjLLJIN_dBa@2tYXeuYq}^PmOmPWQFCB~HzrKl z(%39k@Edhv?Hne~Mx3oR9B<`O4Z5x7%Q56ME z^ktcjYjeNn7lQY0%rDMe?rNQDJ-BdW#n=C$&G)@;54dvZ(^}7))wR1%mdZEW&YU;NJwEMI(gu6yWe$J&D}6WEDg z7nGmBoe#h7Y&+KNzo{M-Ki?P6Uw9=>e>}d;@9Wxqd+W?B@!@eg-P4ntZ8_1~Ufa|k D%PiGd literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/manifest/manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0 new file mode 100644 index 0000000000000000000000000000000000000000..2f4d7efd1ee5a94920f82cbf1982a8108d24a2fd GIT binary patch literal 1006 zcmbVLPfvp|9M!~l^k9tMdtOMq?uMgAw162f7Z=iW&}|qfvvekN;nVDFcK7Jj58&0a zpF%q4z_IAujnen0@BLbwx#i<_$AYFm(i>`f1C;3^gd3J?mQL9LY!pQx?|V@{;hc=~YC=I{IjO}y zn0Mr5UJV4aM@EqVb;-=UtUN5caRCh#>)J2n%1CE!;|e-U(9%T?zhY6k3^M4}+}szz z7DU_GZcEKwo7PRal4aVjPFEXUiTgjYmmSStou35MKYUCDVFBU6&fCfB%l$i_J@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhGs^w*5BsFt;urghYY#X)-Ww&)Yz*3m%$^Hi6MzaKw^O<`+xZzmFlN&z58FVCCA%I zYu@8~xlEed-v8cLD=%-KAyk_1{zUB7wF@Vl47eTC5jIgk($93M(&ioQ8nXo)m>3Q` KFlG=zw*>$)wQQCE literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 new file mode 100644 index 000000000..6e93d2b42 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/schema/schema-0 @@ -0,0 +1,74 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_vector_boolean", + "type" : { + "type" : "VECTOR", + "element" : "BOOLEAN", + "length" : 3 + } + }, { + "id" : 2, + "name" : "f_vector_tinyint", + "type" : { + "type" : "VECTOR", + "element" : "TINYINT", + "length" : 3 + } + }, { + "id" : 3, + "name" : "f_vector_smallint", + "type" : { + "type" : "VECTOR", + "element" : "SMALLINT", + "length" : 3 + } + }, { + "id" : 4, + "name" : "f_vector_int", + "type" : { + "type" : "VECTOR", + "element" : "INT", + "length" : 3 + } + }, { + "id" : 5, + "name" : "f_vector_bigint", + "type" : { + "type" : "VECTOR", + "element" : "BIGINT", + "length" : 3 + } + }, { + "id" : 6, + "name" : "f_vector_float", + "type" : { + "type" : "VECTOR", + "element" : "FLOAT", + "length" : 3 + } + }, { + "id" : 7, + "name" : "f_vector_double", + "type" : { + "type" : "VECTOR", + "element" : "DOUBLE", + "length" : 3 + } + } ], + "highestFieldId" : 7, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788853436194 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 new file mode 100644 index 000000000..e07a54596 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_vector_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "0631a4c4-fe4a-420c-b6e8-8dee879154c9", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-5a2523ff-cfae-47c7-8bec-c476b104f406-1", + "deltaManifestListSize" : 1108, + "commitUser" : "460bb465-4bf5-4692-92df-5c54b4d910dd", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788853436225, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d733aa4d5f993b4af7d2a11de590694bd6e38e19 GIT binary patch literal 457 zcmaJ;OH0E*5T2xA>7j=pPS}+_M9@&A4~z7m;9*h}#a9~8vYX6D9!mAhm zo8Zh=!IJ~?`ex>vS$0TS4+GS=b&3jM1)%xS{fZkofCm6RYmF5F>$uk9mXK9h9SEue z>4~8ECu6`d1nWFq+W-tqs}EYu$M=(m9DP846SP0QgGd=`YyuX2(+3qAL9r33=*^2h zz2rQp|F_4q_-${ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob new file mode 100644 index 0000000000000000000000000000000000000000..bf82127daa9c634d29e7988f784d2b260c119126 GIT binary patch literal 100 zcmX>v=ojJS?BvVH#K6FilbTjil$oAUl9ZF5q??;qpbHY=V3Lvs$%3#o5QBi+Q_n-^ XVOk+V5Kv=ojJS?BvME%)r2ql#`#Nn^;toSgC7(#5R(Ulmbax05J%>+-Y_8JWL-%h=D;C Q$~gVT@?k-t4M;sB06M4?@&Et; literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 new file mode 100644 index 0000000000000000000000000000000000000000..eb9d186f63ce003900e760eb04d3119314cebb20 GIT binary patch literal 2333 zcmeZI%3@>@Nh~YM*GtY%Nlg~2DlSRPOGzwB5d#YrCugMQCSH@TRw}71NL8{@P%27I z&M!()(os;#OUwm}_$KCMrll5_xaO4ih+uwtKo{`!(4-cJ^lSa@-U+^ z^GcLzbrg^kd3*Y~;8zjg7!=|ef@V-sWl3r=no&-n&fcye_zes34{`K~ho}t3uhh-c z#}!EhzVL8KEJ<|B%t`f4ElC7v!5SPO^Wyy+eUS}SDlRF?%u7e}2v}vXXB3LcoO}#* zLH^;9%FlP**=^ zU``12b#e_tPLtSDE-*=wrX$$d!`0U@-V>!z!D6zHYnZDKvhA2flWRmskYhY3Ga{!Q zL~LRt8F+3+O%={Tu8yGcD?Y>%CG%rRg46SR!N(F#cVG{D9R(fMbXURzE{a zZ$bvSc?JcCfa3xsS;8GlTttKic>-gU${q%mV6n9fW}7z^8uxlLEHw39aeIbe2-_KE zjjj5B6M`6@vN0^;(b8zpJj?jG(T43Av)(bsgpG<}3C!i+#XL2<*t2hYd!J%9m35b0 zlQwC|d$z@r?9v8?3`~yK?#?;1I_h@yn$uq1)#Y15y-$lBRVlb^AUyfZ{Q3WxjxUt% z30re4Y@KY(aXzi$JsO8&xuk>pt383JEhvt2h?KXEDfa+mOkmlDi<| m#^ImLPq>>kbIZKWajetkT%5pm{8zJ1f>#5lodSCYy2k+tQQb`d literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 new file mode 100644 index 0000000000000000000000000000000000000000..8608500a468f8d6104279359aa78d8388fc57f48 GIT binary patch literal 799 zcmbV~%SyyB6owVt2_hn{OV&mA?!q*Kg{B>P8O5Q5Oq*(@7fCb1ICSSid+(fvmoaoP!*8Tv_8Q!iz8Nuew;AO1eLWYW=U97{Ajl1$Pn3%Zke*aNQ*N8N>yn< zeceFBgcvokLhv}d=pr5cyhSGY3TJwlKutp$k=af&_Xk=-26ETr2hetPw;GE$E_9pe zzd8G`NIEWen!~zDhS2cZ-H)e&72l*_c(x{KdWJ!ImS?t_9)WCT|6xBpv)C X!!I`;z5C~aPJCH`$@_{t+u04sU&4XYj7e#+t87VONNOggkezA4P}c3Phu{14EuZ@sR*Yb(rnLga#aE>$MGBpvX#l)A83vjd38bl zh^?rC>R$^eDyvSW|K{w=BAD~@Tyv5jgBmMV!_zl=7`anYB=%%MQm%?3D3+^oXy`g+ zcTCfF^gTUv9M}u285miEosJl4;IH8|{Gx_**fFuc@*u!#c;U4w#Y0|Vfx!KtKe%b7 zd>jVHLk%0wwhKL>z`;K<8Xl48UdPi);_kJddVhS*ypXq%&HL?-WQ`|@w`lCl8a{QJ zL3$PyGHE)8GFf_WXE#F^V3VP>EXoz~X-3m^on{_3mWXwNB>azzkp#1xj~+y(dwc?v Cx(=%V literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 new file mode 100644 index 000000000..e311af948 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 @@ -0,0 +1,43 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_array_blob", + "type": { + "type": "ARRAY", + "element": "BLOB", + "nullable": true + } + }, + { + "id": 2, + "name": "f_map_blob", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "BLOB", + "nullable": true + } + } + ], + "highestFieldId": 2, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro", + "row-tracking.enabled": "true", + "data-evolution.enabled": "true", + "blob-field": "f_array_blob,f_map_blob" + }, + "comment": "Generated by pypaimon 2.0.0 with ARRAY and MAP.", + "timeMillis": 1788513128455 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..8e955da73 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0", + "deltaManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1", + "totalRecordCount": 9, + "deltaRecordCount": 9, + "commitUser": "297eeaeb-5c62-4b3c-8f9e-950b09612bc3", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128459, + "nextRowId": 3, + "uuid": "d2837b32-27c3-453e-8470-ccb9d176d3f0" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_time_types/bucket-0/data-92408aec-37d9-4505-a929-bd393a18b9f3-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e689b55e13640fd7417161de56e4561f9f37a2e2 GIT binary patch literal 1548 zcmds2UuzRV5TCnj&nQJ=agRH>hbZ=_VB5sd(g=ccX`7}-ngrU^P$)4?Vr*j4B$`ru z=(G6XM-V@Opn@NwkK(5gzkmqN?2+b>Mk>C!WasbBZ)RtQOAZThDP%<6$P$AofSFJ8 zKcis;&;UTEEkPR6x~)zb(r_%nR!gwogqeUw1{EE$DLSN6L|9(SN_~VYt^*w3`al*r zC2s~X*=h-;O*qmKX zn!>9rkC4 zr#dqp&rd@BVQ3z`Jok@Q1(%umPL^n8&b*&$O}e?y2l!nmo?75P3fVmM2po=h53FdX t+&y?+?ax*^o!N4?+a5$;RJ*-K`&l%eoQuuH?H&381b=W}0OaU@;1_RhIJf`+ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-d9d9349e-8d38-472e-8bf8-341d3b97a51b-0 new file mode 100644 index 0000000000000000000000000000000000000000..2a42c2af5ea7a0649a2f2a34002c80bbefefb109 GIT binary patch literal 2210 zcmds2F>ljA7JeRtP~f1@xg0-i+lSak-K(bpIXS|@T3!gT-3lFHAuj~TkDsV0aGSVIKtRHi=ta}T zp3|n_z0(26+5x&DSGDN<8I>Clp|g-@KwGj_8IRbObST3iqQ(iq?IEglv%Cf1crsXp zl4A9c#(3z4tN;cvPLB#AXv6ieiV2F2WXcioS(>D>q(Ym7Zg4QvNQ^Y(2Pvf28o}w+ zU8`I*wN%d;I91lHEpVS{M#$1nm1@(#D3fJC39|By6Q^gNh&efyDNt1Aj-;4?W&UHO z(;+oeQFp*8cndMw+$-g^clsADYS#l~=GME*x z0~Gf5(P#oSs7qF~jI4K(X^vQCu+3>#lyx9Q`DH;lwfq?_j0>ec^I8Y4jXc;-VB-2T zZIOQ@`mX1>H25;in2Oa3Pyhs0RzeJ}?Hha4)+r`H$5g3+N-Ts$(;Czm*%Go|qD-o* z7j0EjFSFs6utZas#K9~JU2TOO&VXo|DNVYE&L8L+{akEIa5J;#etJ(L+K_bv#p6UXa2*bS?*gd$IWvy zO^1Ht#M1K0QW0TWEZlZV;>zmMs)$O#7cKbAI{foZpE#zFyzhqqjUe^0R>XyLY*uX+U|fUmxcF6-(&z_7sTXhJj-R zaOP(`06hhyG^tN8O{0+IVi3g)GeKo7j947x6+fCTI5I?*t(k_b&(oq*fU>7Fpn;|% zVnB?VST1;!UUreTcF`i^Y>6{mOrWYFmB?(nnfn8+A{{v^@~6;pG^ZMi*ba1>>AyMq zut+*CcbdbRL55Iw+ue@~!SZiXFg#z8G+bRLebY5sO^-l2iQ~%ASND{2T1ab=U`f5x zIRk2cUg=fEfQ57ti)!UEp!CZ2TJ8P`!jNd{ANu8_mU-3&7fpRWxcpq}3rPq6$neUw T`^^LA;dN_Ayga_m4rfpADHZe_ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 b/test/test_data/parquet/append_types_compatibility.db/python_time_types/manifest/manifest-list-f93b709b-b4cc-4d04-a036-e41d4398d1aa-1 new file mode 100644 index 0000000000000000000000000000000000000000..f763a89f9e70478a4f224e36f61fedf74d68a5e9 GIT binary patch literal 897 zcmbV~%}#?r5XUL)&4aNq#$&Vb;6eOqjo?XAFp>Z@g;t}%&C=Z}hJt~uNo{x$51u`F z^9_3RLGy&iR z{Zt>&-Z=qVB@mF7A7HRl0v_t55e<1xn}l@7afwDKN^)(7xHydYqy7g6n{wSyYI>4? zS93fHqNUOZw4yL4ILxZ9u$r4PojZxiiXh~LO^N;+V zST_|j9t#?#XpXA?&S{Dco1)stc5nA63BQ=4i3fiq^@gg#YOSGXnmz(v z*L7o$KD$RjfO?5*4kev*r|%3%{j(w~PGZtVU6;lyPXr<>%C6O@?#h}773mlIlS%8f z{Ve#=)Ue=qxy%v=ogV+l#-d3SX8Nyl#`#NP>@)elb@KPkXDqRt58r`keHd9pQm7?XP{?b$p8V% Y119`957a0C<+}wRkt`}!Vg_;<0afu9_5c6? literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_types/bucket-0/data-991d037b-30b6-4b3f-89e2-d97896fb4019-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bc1c0ee2ba3cecdd2eddb315e28b1e2c5672ef5c GIT binary patch literal 21841 zcmeG^3v6W7b?=SGv$MwRkDZrs;*bs6eK`B`*k13>5E6Lf8UNPyj*ZtIPe|RF@z@^k z%-DFwUp49ykxwI<64W3{QAH?06p@IgDTN3{8;T;5(v(sZA(YT4MTkUc)shsUc9ZQn z=idANZ`NL*RiuvA&+mTD|8vj1bJr0Xi^xXD2YH1ohsVS1t*M#~Gcn;c(xyW{mAwmx^kfjuZj!ulJ^;q3LI zMjq`qd?Dwc`_iA97&x zV{e$An4*l97iPBRi>C`Q8i8>T2{uNO7QKNgi>!jbk~x93-%`q}MH zVRq_6VdE7D8Vysw7s&&mP7BFBw}TZ19)IiKeO=c)J8)O()u@Z0d+am9jvnB{(dV4? z+r^ceoAnMm(^S}xz3ho@X0*KIb#t@xqd)r9AIdJ+gJLACzoDBsd$&XFzdfmmk#JDF zAvN)C^lpdR%^!c@_wLvia3uI}P>dLFNX^;19ct3yU;g7~&p!D+XTXY)a8SGNmK)c7a-$SD;B(t6obA(gg z<4p-b5#i{C4yvncL{CesymDKtu;K%P`$MXE%Z$BH6TuULh<>w{AkZ|*w9(+>r~%&M zS2!U6vzhDI34o>oY$VKodm_GE{bOa^wax(%T>Cv~x*12L7rQ7%x-s5>V4VfSp}YSJ zaxUS6K>*Sn44ph0YSzOCd-)Mgu+49@g!1-%iF-ntcFql7`z-fx@(2?d6dOG@i;tY0 z(8*w%oD0iYFSHK{v~E6(80`od&z$q7Hadi2EiV#Z1roFiX$#$83H|+Fkt7U-%1Sv` zSgtn0YoWYU+W?aG_}{_jgd1swAZ3e3OA;`t$d<@#mM|TrfhCNVAfj7ho>`*sIc^E6 z=S^gg8x$g2D&`A_C%1_9xGr?U$eSkvZ4J;JZlpccKI zE|;@4pk&VpK`|>FU~>dDn-A{hYCRmf(UN{bf5Dd0rH$JBBAw7IWtVSTDIhJ1S>ac> zsyuhUS>|p1cUJD|Dzz4KmC$*p?M+gi zdTP66ui2OEqHLq4wK<#zzi!zMeT;27kd^H6a@K7D3a}}Vw{e0nzfQ(lg#K?^-{h$7 zLH-UmJ2o-T@PK+RtZx$KU9dhg3+K1xVy;^X2deq`LOB~6mqNZnQrK6Vf!un?7v4EL zom*Ti6|xl^pmwcT*(g?OhpruZ?3&)H12D2jA8DMOhEDjV3&4Yb zt~f`{AT)Q(PS2JWvoP4i24@yma;1V4YKCTDb{fCKI3x|VA`$EYZuoZ==3YQrtt}TT zbGA-?mQRpZ5u~{XXtP6h8s7^oIFz2ADOB^z#iiQfGJZ$s3;DuV&q60{MW;KaBZrzG z4f>AWicjQf7cAeFIkD+ZnBx!agE{bIbN0ZTT_!wHKItG$g7k=p_Sq&0Oe#%M*`xz7 zX}5u?D>~f~1$qZVLof@TUbDJz))38tC(c4^f{JIdwZft1A>2Tcg+Sa|u?#!#YTww!T9KL)Kv>TvD_%%il=LK*S+ zlv7tlAc#=VmNclV9Q5c0V8ypZ#bPWNiUksABw~BZ#zBKH!)?V6bScn+E&ygJ$O(OL z%mYVd!V_59X{_cKmkYLZ1E7#5Q2>T}Ybr3<2XdJBcN8jFNX4)Wpw~4OR>6ZQKFvW5 zP>}a$3I#^Yk1ig}&lgKG%Z17o7Ac>iU{zGzCbCPE0n5RHW4Ly zhK=hHNJ27Y{wb(##Ts=|0_0EBy@Blt^zWT>lt4>s%f%a2sWs$l>6QX}T5zcjH7fz- z9$ER8td6L`dKY0@j$&DgnMr~p7hJ&*mDF-5Ahnp*dP0~|a_Tekjc_x@d^x5Crt`(m z%mug{uL}tFE?N1etiIw?AMyb^o;PCL*%6YNN(vLwFZ$J&8vvTd$-1-Osoa>E5syYzaH2yzmlN-{D1 zVo<#w4QSI^6n(nl?8%iKBYCXd7h^#y>tB+o-zR-*~oOf61W7BIy6SB8MPePcHwbGwI99~~?PkSeG7>qsxVp`Upj7 zOYGe*D;H$-C)cQVUW3SNH%saxf+gHm6%)#z`<3&J>Vt~<{QlOH{@=2@Anp?H;%;lL z_DR}hNUiF{7WMg-R@+^WEu(@`o=~y0c<%?)pS~Z0*SPzE!E3ZUyic%b+ei>%mUmui zRqttq5O!Yozz}x!ZiA4J;2BwYT~@zwt@`E%TTOR6Os_!7{p*kV+OltINUYEMl*fGP z!w1xtKh)~Hy5Z~r+FQBuu^mpjq9#sWyiUF2y7!(Fu>kVznaOotKd3%_5M0N&&w;LE zv^*>-7Ax!K&g3}{s_JWpz;m{IKfqmuu5`LgbbfQadf|GYep{nfBI2Whv}AuyLx%ry9|2WMS{wAW##9x`s&Bj2R;VQh~FpU$9MywTzWl( z$3Xrfcjd0t9}>59la203+VL-YgQhVA;{F{2GzRDlQ=H8nFJQ?nGE#Cu=?n*HTzkw zX1J5&;`Hkpgo8l~2|8K?I!($0vhpnKWzVQ@kF>&UYD-tRYT9L5866yUMcq zgImO*n7FH9I~3!=Uk9h@JeANsCo8{zbE9$f!Z@S>XML);-`8-Lh?7s7j%M{kS&QAp z(JQg=vaDQ`)&H7MADaLcoYylvE^WB8z^yfZSr#lOzIyxymA7Q&KA-yeDfN*wI-h1M_pC)n^eIMu3a4X30Zjss+?QZi=RLTcHZT< zavCP(ep6OnhQ#$d>JL9jVPe0ljQVFO*<%RNoIM;0r zH)~^KXC`l8_Z`CLBk+z4QaVmpEDk-1@*G?#$REbt1Q&`zA|f!C5&1*B_3 zFqFi00VrzmQIIKqwg=#k00pfNIMCuzkWnjZX93?Du$mY~R1p+2K8jo;``XkQP!IHL zV@a5(X@mG+s=_&l2tL4*7B8neJLdDfvCgUV;L_A&H+E{12;VWj+TIRw0;N8D_TYoz z2NeTSB!G+n7JTiSSefY`T*{<}wIaet1$@#S9u+U(@wGoQpX;9}Vdo0MpAzs|l7u26 zoIXw>CgQbBdTc(^*BhB4bcu7P0Q3aH6MLi7OZ@Auf~o>=6TL(#0rMBuxV*iL9)Z#; zkwbH7@#ReVoHmT}J2(0K6@rgbCBRiIlP9>4j*t7fWim+AQCM%IT4wf9Gjo% zW8$nK|EC21Lcc`cBrlpa-Z?e7-jP9h&X9i2MQ1Jio+S8aB&V-==nG(kzEBnMQm!(# zk)52_)cO(rNJQ`-$LNPp!G}p-M0%#&Tb-Fi`FrMYens%Boh2p>dmuD?$p34l!f1l= zml|U7jjmCAkP%Gr$Ml2xjxS{UO4Uq1$Vlo>5P!oue1!Nj!BZ42Ax&~(77y9z>mWhR|Lzn-1xE7dSImsiJ`zs)6d`Qvr;mznIIpDM4HGFoB)>8;cHi0Zzqe-nSF zQ=DF1|AsY<;=gwzU+(SZdd|ln-%aZ9?*4rXvf}FB@nM87>hO6oav#oj z9({atfF`-iTZ&{RY5CU%#C$DDLZ$Wg<@}>3iS*XQ{O$M$)*)I?1mU9sp4TrF^rrmD z;)jGb*26QXpY}Y$qYW+}f42CICpy7M!!edmyz-A-zZ##fY2>t8RJ8f}!t99*u@9Di zTDEhdTk9JKdPn92y#(`jTr~N5MEQ#JPo)y%E9Orn{dbnvYn4iPs z+SDW{Z+P=0@W+;7oYb>$g4>VPpXU66%}h0)9Gl_#Wcl zY=4}DdkQVC!*kymJ;Lfw?!P_5ye-yGcYj3N>+9!8BwsGAWIAHS?BvW+t}?0>k)IRe zTpo{n$6UuX->o2gyMRw}c~~f*BlqFPV4Y|zeKK2rlef~)0d@wfjkdB(7Q5&3OSts zd`I#}c>deYeqa;%j|%!*+z+J2k^i+LJYVv9h2r1j|5Rl|d^=(^y@^xf;3E`2kQYf_ z%8wHhV+pN)9ODD#Bu|qP%=;^pm$pW*f9)Dug8dD|S8omWvDKOWrCCZfLr_i;{D4dR zFZ1<~R z46J7?_9Oicn4U^l93!AUK?&zytDyMW34}WzWfWmVe@~zq}H8NU2>& ze|>@TlN@0A&z>(Zbnv_b_rZ`m;yFm&=wEP5lH&RpW%Z*Ph`LZ>b#AM$UNZyn^@0Lu-ad@S{lR?fAv z{36yn)B1qyX~JJj@-d$9Xak%VL=k>Xse<^+q`J8OF#X%tW8pt0{jaAX5~1VkMm9Wmy8DSyNPFWxiSrFqe!o76W}So^7WBIiChhFfEU3J<`3j|WN|8u zsNz+mcbm?y$Nq`?(^TjDd>-CQI6syU^w=Shqdq|(iqF-V$-ye_Yr+3+LH~mn@Dn(* zV*E0A@*HzKplk@AspdL+X(Pk@V~YC+43BEChhy~`KVKk)xado-WdKS$CFJENaaoKn zvX{qxisw5~pZAOTd_PqdZC-CX?9KI;hdoiQZ=Grktyjp;4+V$C`X=flFZ;pzSSc|a zT2GS854u1i7(BGT^x@GVEiRx8-3dn7sVX@UTb~hz9d(jF##ZqxhMipjDh>+pkYEWv zh(>(TAOQn7;dj`d{0q)>$k>p)=J1G$kw!qazakV;_yGUk%g6FdOUK~i#Nyh4RrtdT iMfk&p9WnUJh8^%fF5thw{nq`HejJ0p$_al=^?v}GI!3kt literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-41694715-8b35-4d34-86c9-46164e966b5d-0 new file mode 100644 index 0000000000000000000000000000000000000000..5bf90fd6645c39b2a242861fb95c26c0b060b8d0 GIT binary patch literal 2547 zcmeZI%3@>@Nh~YM*GtY%Nlg~2DlSRPOGzwB5d#YrCugMQCSH@TRw}71NL8{@P%27I z&M!()(os;#OUwm}_$KCMrll5_xaO4ih+uwtKo{`!(4-cJ^lSa@-U+^ z^GcLzbrg^kd3*Y~;8zjg7!=|ef@V-sWl3r=no&-n&fcye_zes34{`K~ho}t3uhh-c z#}!EhzVL8KEJ<|B%t`f4ElC7v!5SPO^Wyy+eUS}SDlRF?%u7e}2v}vXXB3LcoO}#* zLH^;9%FlP**=^ zU``12b#e_tPLtSDE-*=wrX$$d!`0U@-V>!z!D6zHYnZDKvhA2flWRmskYhY3Ga{!Q zL~LRt8F+3+O%={Tu8yGcD?Y>%CG%rRg46SR!N(F#cVG{D9R(fMbXURzE{a zZ$bvSc?JcCfa3xsS;8GlTttKic>-gU${q%mV6n9f6E_CVn)GhkJC&7Ew*9S(=Ck}@ z)7Yy2H=&7XDldasxSvVG@=Ru9gK|x46VAkviJ9kmb^l#*Q##r4*FOA~51$e9hT9rd z+vik@{Ne8YcJu$*)3a}@usSl>GuSZ%oO}7FM_iQE?Cr7+Rsk7ydEGuMOd{yWtwxs=qk{c@>m{pF&0yYq+iD`9Q_HGg`R zGp}dZdRTwKr733hi|u4%4~wa<`1DYU%`#T`#gS9*8@9^-U$pU9K{B5}SuZC;;`SMd zH~r=uIR1XYW~YgMN(wK!pERMz}>zT(ITIj&(_vc619asjHo)`EtH ziHHR;YGb9~Nq*Bsy2iCaro|d(`ItamM>>(&Zaen}T1O`GHsmj$;~8E(7I8i3wbOrd z_F<89T<$c74U3GR>8ssOW`dR9q+ocpA!+%hNd~rWDNT<-K26iw(O37Bb2^jO62Y=& zr%MJj{(>r~ixG?IG!^yAWk3~F>RR2w8N!el`XBn`q>g>5f~%%JA6$P<`9jjcKQe-9 U>-^}@%buO^wD+>`51!uM0Uf{cEC2ui literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 b/test/test_data/parquet/append_types_compatibility.db/python_types/manifest/manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1 new file mode 100644 index 0000000000000000000000000000000000000000..d004f2bd825754c794be390ec991d47e4cbde2d0 GIT binary patch literal 899 zcmbV~%}&BV6h=$w&O{Sqj4>`qH@ct-p;T~XV@qt(7NrG^1}8&j(3(M3RIBmn8y$lEDJhzG{6|u+sw&tkSM}J?r>uMJ zIH9A@>7nPrPGrp{xHZ`6h+zl*>Onm$BHV=64hdHt1qAgVzE)#zU)DsRNWbWhZdxfH z#li7V!-ikm#hy^);2#d9KjN", + "key": "STRING NOT NULL", + "value": "BIGINT", + "nullable": true + } + }, + { + "id": 39, + "name": "f_row", + "type": { + "type": "ROW", + "fields": [ + { + "id": 34, + "name": "nested_int", + "type": "INT" + }, + { + "id": 35, + "name": "nested_string", + "type": "STRING" + }, + { + "id": 36, + "name": "nested_decimal", + "type": "DECIMAL(19, 4)" + }, + { + "id": 37, + "name": "nested_timestamp", + "type": "TIMESTAMP(9)" + }, + { + "id": 38, + "name": "nested_timestamp_ltz", + "type": "TIMESTAMP_LTZ(9)" + } + ], + "nullable": true + } + }, + { + "id": 40, + "name": "f_array_array_int", + "type": { + "type": "ARRAY", + "element": { + "type": "ARRAY", + "element": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 41, + "name": "f_array_map", + "type": { + "type": "ARRAY", + "element": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 42, + "name": "f_map_array", + "type": { + "type": "MAP>", + "key": "STRING NOT NULL", + "value": { + "type": "ARRAY", + "element": "INT", + "nullable": true + }, + "nullable": true + } + }, + { + "id": 45, + "name": "f_array_row", + "type": { + "type": "ARRAY", + "element": { + "type": "ROW", + "fields": [ + { + "id": 43, + "name": "name", + "type": "STRING" + }, + { + "id": 44, + "name": "score", + "type": "DECIMAL(9, 2)" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 48, + "name": "f_map_row", + "type": { + "type": "MAP>", + "key": "STRING NOT NULL", + "value": { + "type": "ROW", + "fields": [ + { + "id": 46, + "name": "enabled", + "type": "BOOLEAN" + }, + { + "id": 47, + "name": "event_time", + "type": "TIMESTAMP(6)" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 53, + "name": "f_deep_row", + "type": { + "type": "ROW", + "fields": [ + { + "id": 51, + "name": "children", + "type": { + "type": "ARRAY", + "element": { + "type": "ROW", + "fields": [ + { + "id": 49, + "name": "leaf_id", + "type": "BIGINT" + }, + { + "id": 50, + "name": "leaf_variant", + "type": "VARIANT" + } + ], + "nullable": true + }, + "nullable": true + } + }, + { + "id": 52, + "name": "labels", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "STRING", + "nullable": true + } + } + ], + "nullable": true + } + }, + { + "id": 54, + "name": "f_array_variant", + "type": { + "type": "ARRAY", + "element": "VARIANT", + "nullable": true + } + }, + { + "id": 55, + "name": "f_map_variant", + "type": { + "type": "MAP", + "key": "STRING NOT NULL", + "value": "VARIANT", + "nullable": true + } + } + ], + "highestFieldId": 55, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro", + "row-tracking.enabled": "true", + "data-evolution.enabled": "true", + "blob-field": "f_blob,f_blob_descriptor", + "blob-descriptor-field": "f_blob_descriptor" + }, + "comment": "Generated by pypaimon 2.0.0 with scalar, blob, variant, and nested types.", + "timeMillis": 1788513128436 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 new file mode 100644 index 000000000..4efdf367b --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-0", + "deltaManifestList": "manifest-list-f2665def-a1a6-4f0a-86d1-3bdcca9c333d-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "4660bd51-f9dd-40a1-9408-3c83d48e2336", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788513128449, + "nextRowId": 3, + "uuid": "e60f3846-91eb-435f-8ef9-161ace2622ae" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/bucket-0/data-408a5ad4-00a6-42fe-b44b-0680026a4cdf-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d39e4856b6ac9e4b8e52b6467e069c2be400dca6 GIT binary patch literal 3469 zcmbVPO>7%g5FW1=$4#8n0oGkd7P5-6s?@Z0{z`(1>bG`clBQ{5NE2H~jh#4-ll2d_ z<0PDlicpavgvy0uPZfe5dH^X04&j1&(9HlG03hHef#j0pKCb1GTme}msVvgI4=yyV zP{7enKcbxi0(+#sKFPx^Akg8E9R%T{soa^@X`ENctj8%KQCL!^2xht?e-H34HdJW1 z<;1y&9~wU|4H1hM{UrBDDvP8~?pZ4obN9MGImqaA-upo~f>nO1v;Ky3Kehqvb^ts$ zYsX4bStNa8wN@{}y14zEpY_4V4v2I*;pfjqxYOgZIV(wJk@ShxTAjqYC{+$bIQ$`| zGw}D<&x|M|9^jXbcPS_E5BAiJtdtJy+YLUy4SWFfx{ z!-DKxxn0jL)k>AynNms1ri+pwxu&k7#bHqvwS2WE37Di_L2k=x*+RBhJBX^~i`)4k zD%qffCRFchQ;%M8cavQ~?rxA)3u#R|6+Vb~uYvb8@TY+iPHjYs&3BktjV5{_f~sP z9KJss%12J+p>W(YYVWqm(^?=$5&bDD_eCVK-b%W!)(Va%I#5YRas1>`p1*_~*2_p7 z*6IijQBUWHRT6sbwbFbc_`VaAf85INZ2`RN#?yG>189FVw?TWTdRtkIOm!#^+wpML z6L;vrvu9~;|6nnf9#pb^*Rh%2s?Ll|2p zag@}en`AM;#u%HXeYI-f7#sUwG!z=a<=8Bepemr&8x8 zPUTN*so_roQB0jP@yGf@st5c@U~IHTe6)r?(OJ{pDdRI?=wmvlC}YgT8~#}LOAqZ3 zxx(wQ*`I;QdS+ltOU~1FlCB!6@YL}7!3E}>*XP^x4q%(Lk~S7UMSYtPu4>vtp^ZHj zwYNuArj}FKqkot@^k-9lZ%|~^2cI#JgXMJP{f%tx(o(s6DP5_QHiPwSrJ64lgZ-nI X!hsR*K3^TtAxbS5udb(Z+nW?LjbW}1n$R@_C9 z9sC!(c@iDy&U*5)g6tqz5yntKSWyOom!U6d*Sxf`t>DRP-sgSZ_xt&NzMmv!Z*=V& zM7M0~RL07xnSnbw2bo!9XODOvbD0IzK<`Ehl#^dnsq<0FRx_5JrN*L^i45-xk4$Y& z%{jny?7Vw0r>S~2hru(2I!H+ZQDPHB;)K(x|>K@OVGbd^^fa^2P}chG7w^T6J&lIh@>aOoQo~_cQ>Ti zB;aX<4TMSvGY67@BVTWhCXj+6t+>lbddHt;lV$4L9CSrY1hku93S@9LMS+` zL5h&gC#xn(ySl2;mXO658E!s{JB4-}#IoSk=E&jHi8`55yLa&XfhgeL#S{%ELwl~J zcRQj9R+OMOi-0Zb#dbD`X_3Vf{l^XC1WOkqg}bkNe?0tfsjpIbyS#FEy?ecD^yTT@ z=<`Z<)w_Mh(!ZHcw?2yH+1m?iGi!st9=#b^`FwL|e}DhkXU7h1 zJb3(dXtl)3t6$3L_t(mwk|p)Ri;dE~((w1q(F2j|edjh0?JS?@IsWthaJ0Me;olYx Bv6TP- literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-0 new file mode 100644 index 0000000000000000000000000000000000000000..55c490337ef746500806312902704a9c1a08f584 GIT binary patch literal 799 zcmbV~O-jTt6vq`@i0(vOm$)4SUAi(%Q!F&C^kWo<5;AS7m3~N?5yzod@BkjcgShe* zv-A=+)fro->fko-mw(>-y;!50^^IM6$Mb_A4_UDDPzaiZl!truVG&%jl)i3HftcJf zaI65%f}Dq-r+|#6^$Dg~9I--Zal$YYRMw)HC1FwV!|8$}17th8Wy<+SK<&>j{i+zUh>jCcty~6_e%W5D(?3NR5?%d6zns*z&fDOksm}+OpKE;~>EIt3 Wez|t|^nCwzG<$qG(e`IsoB2C5d-E*- literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 b/test/test_data/parquet/append_types_compatibility.db/python_vector_types/manifest/manifest-list-f770cee3-649b-4fb3-88fc-d8509dbe0f64-1 new file mode 100644 index 0000000000000000000000000000000000000000..3dc50f7c3ff42a43ab68f1b21df35b413a78cd74 GIT binary patch literal 897 zcmbV~F;Buk6vs7a60xGfNyC^zUEh`e#1XUWueCpZm@iOi3gSkJ@qNGU1g#4aR zHe@5b7Ss*F7)<}o*_TB)=I62IR9S;nL8-*2H`|o>LsC@XU_w%>C<-i=Dq3vl8gS1Y zCv@~ZJ@P!%j;z^~G&?)(F`(nG9@N7kCJl7%&~W8ZKu{0jYt@T8k}5!1{6l|q({gD) z4)%xYHXLmidqR;D*KfJxXKHVHYKiq7+v)75m+vSgxvBom@Q^})d_o;JM zKEpT6>?&uPnCG%9>lRnYWD6YTSIj~#Tgc&@$?XaCVMH+(jOGq0+k1wq0)jK+6ru0 z_Q9;i2i<=`W6Z|*xbejo6W{#96l7^h z=)w~3?g&G$5My&cExwEd9mX`q7>CCPk62E=V?;QpJ_r?7k*scH~ zzcBV?5HjoTd41mLbezF_JfX24zTO6`h)Bf5-pZ-G#>}Rx5e@p?(|oT z=@8&Hj(dLCJFSyuZ-^MU8Icp>78^n2*Gv)d){to1xtK?2KzvigW6<&$B{kJ=aBvd^1@6o6R7of8Iar9yJ?n2kuVrS!VIpM`gy6 z+zK8a((N4LMF}p9BoD7ak)#eZTiu@EOay1SHPLVz?Z#OTb0&`uL8xH{PwL^A%9wzF zPOH%sIwmp8ciI@tvYjw^3B-TtxDOi)6baiqQ5FV%NoGvhNJj)s!~D?vT#M$Tfv^%& zDQmJhl2%#M)69Gvg!I%FI6v0n+*%nTJ>O zIW-MxHg2pJx*if{elY>%{_u;`|CLjcQ^8NoXIk{R9{t^i?cSnxhXZeon>P}eAp$tZ zDEW8D;F$ppoB#MsBN{OO(r@E%T0vPV?N#4X184neEaFP^*QJz2cyMh?*_Z1latn`O zmoi1?V1g}W4lOtu?qKc0lg=SRT4vyzp!N{@5-gcQ5DFqIQ|r@?FDxwc={t*ycj6eo zi8j>tR{n@Suv7|xReWI|;#W^FK1=>#d?u47n_flqr;y5Kty$l!xOfU6Wv`!t$1~^I zW%OU?m$50eG9LrdD!H6QB`GDcO1bs3(!i-!9!q(ScKaZ))I7@GZPhpKw(tOx>9zY9 z`I24=pIRB9pT4rGRN2i|N|oYDt?jz?jnc7G1>&ik+#tR=ndvLNAwPrX;H0L(59_M` z-F>nN^#lA|DggCIQi4tZxGd9sRT6ZkOL{FoLz#quaYauWuiuSIuU1XQRXyxCJ2+}* zlLv{dZaqQmd8i1zo*d9Pi&s8t`G9wgjT@IJtKp?gj3 zs{4j~NO^!rd3>pPqM<@lHVGdHP~47`E${t6z#u+IPh literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 new file mode 100644 index 0000000000000000000000000000000000000000..286b07f38b313aa7d9884d28750cdc6baa602d40 GIT binary patch literal 2432 zcmds2PiWIn7_VixgC4YuIRxkSE(&&45ca3AL)N}+acMh~w5yvVOWU_Dnl@dMP39B^sspl1aXQd(U+u6vSw`+=EX}X@B7~G`@Y}% zeZNE!C$?;D=O<;!my}W{iGMRW87Yb&WjPT|BGK)tv%;R!ippnGyqv1n50&H?%J7%K z5Xer5BJfTFbt;E|AIK;v$tmCkSw4f59G}zhVg=2YD&7VGG5x_H4C(`O*zRH6q{c)sY^YKv zh66^TbindJ^SZ`xrOUPz!^=%sv=_IWd+s^+{-uLI zUGdHum)x_v`SL=?ueG)Dj^~SwFJHYCm#?X(21AJhR~o2=J*`bGOLfcAgPNzEGc)6R zZ_Ndcv;`Y?Tv%B?bNI*WU8|2iPrm=OeZWSMPq0F{q}!w`U2hUH^D;qKR1_H{kF|@)6<~VzPb3$J=V7G{D|^ThXGP}qNQCcq znjr3nAt;XkAEpFU03}{ReNYB5jtEP!CyNQ~Dp;I*L`iWFC9#r`bV*Do=Hv%nZ0`gl z^cko$M;Vj2LJe5A^_nHPCt5+w!B|L3DG4~z4Tvn&ggJz98pPdsl&xPBP+>BIv0M|W zng+Fu)6V%8A~k40X9eu3T6c6uSU|R;IyvkAgox0*-hxz3-9io3aAE!)&>nKaeuZYL z=PRHr*Dz4E=32|lVM_5(BpPrM$pKF`2xQT=#;r2&NeA~s4k~iD#tTY?ncLN4XsSrp zmdr?6Janrk(3yg!s&a?sBn$k~_}aZf*MJRZtS7r}t5B`i&We#t-F|00oA8X?zcq{T o9OVCm_${K862^igMj;uHaI;i;y1DxNcz%7mdw6-@ezb4D0Lp7^(EtDd literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 new file mode 100644 index 0000000000000000000000000000000000000000..5478f74d433e0f9d302c438173b00eaf5c1748a1 GIT binary patch literal 1219 zcmcIk!HUyR6wPGH;KGH7h$|Nl6c=Z$8;Sad+4_x>WZu|zf;s=P}OB|cnmSHTod)%CRPR_ZP+Z#UL*t&)%ENcZ>NP?>` za*k<;S$MrH=D`t3@#`xGfF36aXr2H*9TU(2j071AK@(7%l6;JVqL{H!3yb#=X^nB5 zX0%n3j0hzRbMgmXZ;xV~;nE0G4Ypmzc6BfamLp0I#iPxmU;{}UoY%X%wP zwM_>Nbjyd;|A66?ll-sHY<>R%D95)f)b07sdFC)A5>k{R(rRtdV&j4E zA&eiulQG7`Nc;dEJoq6zdgA0sW8!;n+ATq1KR@|b%>i196Cnja7+z=V}8!!yd6Sa5F&a18j8w{xC)@m zjLO}BNVUy3+&5naee;E(O~K8|jlNR7{-C+sxP5oES`x>Y;6p6J*YD(E(tL|4OsNmxs~&_ zJwbj3523z(($lH71^3hTW7|d|YO+|qGg;XzdK0mB9&S)rDR=&gZ8wpW%Hl$-Q$(GX zd^)p`_q>dks#Qw`uTjh-YDG#^ag;iXwnM4xK zLams&hk8iA=1(6a;_&{e{ zkfiGuD~ZukVXU&)YKOa0vyi0wrTaF@GuC;0r5=|lNpoS6QRALRYYZw_HJ zKRp%n?@~X359xprQ>Ys!PFkv-15Jsff_~W7kP_+X^aD|_wC)F5J^4cH`gImDYo+?b oN2{9`>y5$3tM$hEgZqQAiIK}AqvjI)8{hxOj>9kcJpB6q1S)W?CIA2c literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-3453ccd7-3a90-4a44-a98e-78b53bfa0da3-0 new file mode 100644 index 0000000000000000000000000000000000000000..8a8baf5fbf2d316d1bad2aa08b02a99e05a879ce GIT binary patch literal 2428 zcmds3F>ljA6n3mMNC*%hgt{PAo{E%&RH-28<|bEa#c_&bw}C1;F7_oca-3khMWsZl zP^V5EkYKFD$OMRmy(>)o09Gb;gv8lSY&UKaklKa8k#+a(d+&Ysz0avMp2QM4v~70N zN>{}S`R(*fY*|{@fI8mBI!)aOwy!_8Y-Fg&RByVAmeR%@biiyf#+I%#$z8_Y>0xGx zF|pD$RVK+8sDrH@Qk=!E*-9hw-p1)3(mGuu?SizhflXxNVR-m>Q^UGyF-x1fE)v;- z>p-k>6@g$6a%qOm^hs(bl@n0ngyQ(!HCB{3=>$My$HvwPh-JA@0@8R8sUopF^rEWc z!&W#R_`7Gz4TJ;mQy6k~pum=c+>B+Ln$bGZqZ?J_9)+Ux?nsPS*?@(LEQF|t#FRM! zmcT>5PbNxdnXOF=AqcQ4u~NY1`Dn`H4Larc`A(tGX=hj6IbxZ~)3Fi?z3cfDUvGAeGOE!}38oKfqavJ@HtwzLn z{}^=yQ3VfVAQZ@K$>n7r28xf)1QGn-M^_80pv*!p>V|1?iG!_O?bm-bS5Xs-x+;Myxa<*N`@g zfCmzQQEM3{8)$x1@gWF6=r0yQ(QOzKy9YI&8Xv{8p-Gi^4p@=JLg0b^afjhaheIPq zhMNy)A0kYpe-b*mPaq298?(krtI^&Dc0LJVg%hii%W_Cp!`h}(ydiSrlFq)d0M{Ro zf1u7Ai%8q`zT=0~@%Fo~hwtlOeq4R~VzKlrLBFBrKi&CF)o#V9c!G{ovsA39A{(VL zIh4<4n+vJDl5M6Im4=eaqkJQ!D$2dZY&Mt4F5FFJ$iEcx=@z#4=i>A%MN@Q}qNo}2 z#P)7~q<_)$MY2ZGpZ5;-58Lz=>i&G<_4f}KUcLMlJ4!!RkFU*Ta+hW=TQ&Y?EPpgJ GTHYUxsSNu7 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 b/test/test_data/parquet/append_types_compatibility.db/rust_time_types/manifest/manifest-list-31a697fa-9010-49d1-8d91-d1599ffbd01f-0 new file mode 100644 index 0000000000000000000000000000000000000000..eb1752a53b5ef14fe479b93580049707ce4d0b53 GIT binary patch literal 1117 zcmcIi%TB^T6h#)U)fhiu5_hC)!>-aXHuhm?iw{C3LpwlCUzr(5kg#y;!r$;O`~es2 z{SH6CnToUmAxPq8GP!f^IdjgvcK2a#{|t|4s`zO@d~wDZC5(k>f+8{^QMGbhJRdzW zj*|e>;3O|He?VgVe9#8TFp5BJ0{A#1pbjYUQyPF8NN`M8hJ9H~XiWPqRaC_xd?`KYja{*SVUiT?%^^ z4?crld;t-kK)m__zJv!61SfHAvTa$`f-r>PpYQ*^eBb}K`$uaV4LoK^)lUNAi*Gz) zBu&GFqKJ%1Buh7o=Nl&}$27n!xRn=E|B%G^)%9IKkD~}wP63~c2xtOE{DcLd0w|72 zI>LTdOxUoB#r=>}M>vcVTFps@gc621`3o<%4?+?JDQNDV=1k%Vbz#Rb+P2_6*9s{I zQz4y6Wq`+q36ZT?uz;kTg><-xa*P)xRFuqNY|lcPu0uWNbPB$uNFAEcy#n@`)^QD2 zSU`@exdrS02@#=1y(OtyhK;(K>A~WEK!3tXdLcAR+qnYD_DmDC+Ma!xIm{TIh(tq9 z;%va@8wB!bd(&P81Z04ZBMzF`ZcUezN;7v_PoSkCL%(E3nZ-w5>nU_+pqXm6Lkp56 z{>=Ewy+YT7U1+W*yK1XY?X;hlBbmAVlkt4Qb9Vo&S&T>ey!Y+r`0Lw``*-hKU*5eu zkl(L8`0((Tw9}KN@9Xll{TS15KvJ%#^-b0HgDpkH+cl+*>vaWhH;A&;=x?h10j>pD cRcb%($&!3qlBMjdOP^jJ+Q{sNyxppXp>3P$!+K9SL~FTh~bv&VLN_fmmf`cy2>uUQZ5@; z`Kg}WG|Fb!h-I0GK}^pGF>I!ljTqG1%P@3=ScZ)d!(>>t7qv#MURLTwXa}*QUYc!1 zY!>!A=fCf|u5uD)wX3OaSN-??=Rfar{_~%Efj(`ZK@P~(^1&{-T$Xl8QthAD{qMb1 zNlEfalH|wV7W~PQTp{0k7$7_FSFS?^z7B{j`)GE%yq9Jr+w9g>eZ9{I3RvDH`z;Ez z_WGA!eu*VJP%Rtu21zzArXM8R#8s6JvdU2fn}j@}J%^>yq^R z?`&$Z7xh26-tf`SYTkZhQwZ3u0ZZ#-SLjU|pH42~hb*^C-sjf-whmqUWxg$>{=@Qv zUHCD*L-{w7v>mILlZ%At$cL}~*+Wz3U-e%~ zQU#VbG&UV;ZaE&Ju@-KNL}T&x%}IPze#n9Go}Wn4UBCi~Q#0A*;u=PorS#@mROOHZ z;oW~INq1m*YAv76-7K;n`{Y-A4!0cr1HaVx&vpgR{KeeA|Jr{#@;|=)_5ZN*_4?<( z|GQ%=+kSHC?cX}^<(L0u^)JtFe_Hv^dmfzlx3B%d<^TELpZl*9@BOPEy!p{5e|osJ z;ePceW7q%XvuEBoeCGS%8{Z80q`#J4*p!QT`RF!M3_hDe;FTHVzxmeIO)d7K-t+7F zAD;Z9SA3g5Ajx~=dUxWwYt2kPJMZ7|+3)_{_hq0$5V!#;4md;lwsrjIH^1^X-F>y= zslE?FzwWtdc<%%m>)xaez0`1~{}-SA3*T)Z_+Ph+;GqYf|K8bM<(qQ@FA=1Jzy6Qs zhF-mA6P66nHsXj0O4a|^bg(F&btZ<^zw$}Vh(v#x-?v2y?7VB2bYE>N(%4Kgn?Ew2 znoW=As}2UXY>|9h%A_h^*{a`HCYA5?Z+d%uM9wRe0m`Grmq}j5bFvwnf}#G%XVtl>z91{pIKddqB! z)snQnG7KILK?xc1?vq;%%T*2o?)rUQm=Uglw*1*S-Gr<>8!A#BnJ`ITsm&p~Xdv^l zBjEv>`{iS9o_79oCr^KRPS~?ne!$_SyZ=Oz$Q!atbE))Vt_;KU+(Ld0SZ~r*ZF0I| zsF=#8h04;ur>hJ^9sH4_5L3y_%y^nSCN)orV~L$%NfPs>Sx(`$W%o`UqnF-3P+4z9 zgPXCuzeA|atIQ&{CTA9Nd5VV|Qc-r|CbVWou>ELbMX->;FVPXlD-+nZz@m0C@Nh7D zDYp{L(}#*i`WVV}rWRAVenxl%e#}X#QI0s$L-4N%1Joao{N-D>SNZPDugq8Y;ivwx zY+lkU~M=^sMF zr3nXj`GeIslSvYO$h&2KP{9S=_W9_KZUhdf0l&Bzpw6-mY*6wwavCq#vlOtIMLIT%AFk zo6Xl593Dw#GDq@j3+ddE^lCo6m`!Gmq%!lt>f1%JKrGoDpkf<1p%0`shAxl*9kQWy zK5M&H8%JoYHioX%+a#kN3|3oX7tjv(xj0x|aO?J$1n8IyLHM-n62v$FLJ(u^ZrMo`8FkYM#z{Y+AzOOHK^NcjWAh~%m7K#;gFSPW484~6qFfEguA zGM@$yT-ZbFdR-5oKv8otdRkmyXmbW^R~XechGxNS;+~`en0YKf!c(lEB2yqxE(%$o zH>z7pdoiHb&9GH3O|d|fs}@v(m>@70AsKygV6_-%Rah% zZKp}?m0jlW)w|8%jl1OmU5a;i%iq<9FUjWcs$#;vb&olGyV@MS6Eug{?lp(+e$5=d zS7Q!;dY?Ibe~&r5vX_TJWtLce{C@pvkgm>uLmt$by1b8vL1jBjw%=s@s>&LmyVt&{ zE2E08F4UTfFCH*qE*&(7FCLOdbnI^(<{_e+R=$5kPUsV7>U6S5_dr>1u6VV9VS>sI znz-6%()y%HU%?i6o%~F8Z6=%Fzo$7+^WxnVI}sr~RXKSohn{PO=Jqz1 zV{V5zS2lSHw}&buIRGRTKsz%>+j(8`aSIEgcF=_ zH9mB&xZa8D%zWz9L^?OVIJ1zSUmV&agt&7OH@VAvs^L`q{z@G3{qo+LN9fm3i4CUp zR3l)ht8qVI@JleHjO{i&AsuL+ypMtQ5zuz5ae^7RR9R&(>}C9%9AI}@*qTD88bYAC zPwtXy0f*m>;F<`o3;#O^IQ$YESsQAfNaoY~tC|njWauU_459Kfb7-?C=N2INuI8gP zPYCUM0W<-iz8FBu{@u-u1aKGxyfCy59i!o^L4h?7t$ zAmZi$0N5kaR?l$gYg@cKLT8%7ujB)TK&bGq}xaVli> z@FVnXm~V#;Vj3yqwBK&`{3#eQW7jBDE)?WcQ=eN+5XMjv>9LDUQcR1c;-O1c{6KMx z{x0%^63V4Wv6KrEWGH5w4K%n31CdeKVQx+evLd-pqL>uy4*u!@JuNf*fmzOt&o8D$ zQiA=_g3v>+CU zrpP%f#;0d86N~ArmntxC+8d=StL`^A|+A;vMgD_!s&0$-mE2x=OLv$SQET)LX~Pz3^YsSJA$t=Eq4gT)y|z#@Gv!mDOu5*R@n5EGpFU<%C(q$R3fZW zaKBWJL#bIrFz$LKte*FsXbIH%d}uEc9)L2Q6Rp%{l$57W%Zl=OOg_kH4Ca7GFFhDf zLRnWI^Q+fMW< z5onohSL<#Onx7EOx7y_xE3QJPX;g5gRG*fqua!Y6>e-5KlFGU&LgBFtC_K@jOI2w} zC2B%Ts?W;RSIQxk(fGASBvs+vA~c^Rns0XKQdQ|R^|F&GPAgCN)GxNGuWqer@z-)k zxmxnVs8n#1SyEog$T-$?>d|1}w5nD{`#t4(pZeT3^=bv7b#S(Vn!eetFapj~-@Ak@^sr(b=1)qBMPW4h15plQFHxOfwPUAVEai&X-IMifR`iy^B z`2no3U47?H!f$WLZ-86TabF?aSG(jB4%{eTT&D)6a@Mc>z^`7}p?=*`=P@g;Yb28^kxI8tbYEo%31H`J~sC zKwU>NOTt(0RzC+N^T~xZA_8P?q*l%FTcFZAgWZWmjnov=+)WAAn6)yQ-*9DsVM=ASAxY)vT)`2ER@Oz8G-1 zn)5JidD>g|sE^+RPcxqESR?YZ!n;KXKRFM=FF&k%nk|UUiKFAHXc}6 zBO+XQw+P{jMELWfgj+1a?*!E+f*@=>2D3&)xbSWf!dHm!8-v9pZnX$syH|bkUJy1O zu2~}@TzIz#;rEE}xgi(fg8qRmXTJM2^{KCYHFo}ph+iv7+-B?dYSargUyZGA5b+C% zQj**3{nPu@r|#B_jUZur9e?gw={Fk6J?dxt@+y zZ~ByveCn0G>a)K^`~R&y2f4*3rCwgE%<3Ec>kFu2J{*=m_u0!Bn-FIIio@ozPkGm; zK7POY#a@!*CLW~T%tKdCjt6y)e@+~KFe<+*IM(xZ$n5W+mBpzFLAIvalm@cD=EJrgTMGKxzVnxkXL`s=*Uqg4vMQ#xY)nEPyOH< zq&Ip@jJ6CO8GDMlogYT=V>3y4;!%?KTk?ZUR8QhXiJE>_p=6Y&eClia)i3tJ<>|pI zeXNmEUh8q!izn#8cy28zmmY;4{`e7H54r_vMQ_p8QDl!9QNE{MRn;d|D4rg2i*Mvt z+m#o=R(?7yDX$Z)cOQ{McAe|AoXtumH=#W4Q?GqfJ^xM8v;N@Rc!2IjN2xlXQ+l5i zeeT=xLrh6jNjjnJ&dw)O3M+%^h1%Py&UC`RA`##EHk$9$JJBtyQ!Z$zGVSjwXMO6$ z1L{Y$x0}gy7S1h#g$v)2+dWwD?AcmuTso+JeBdi$z>faxP)6pXtJ=!a3ZC75-aKQ0OgQWbJ_<7-!t`?jZ8`^WSJYQ2 ze7Jn;uzL1zO-n_sFIDYJdXdHZ1x%P;7oeAHmA4n+iXWenA7FDBdMHWP4E!pEdlv5> zQJ*^k6;R;oDyXcrU3u|hQvn{TOUj3&f{WjEs(`Ys+3I#lU7?)w;UKA=sZ%c>r64rk z<){`fcX$z1s_hx?bCUiOgQQLR0+K>thYc(TxKOXYU*}BsZ~!a$docp(ZLg9@$}>4g z|8mkPeYTDQ%x?dU$(C_HpXpBGg$4C$qxxhcB&T;=#J2*;tt&4U4j2ru)!!z$f0S}IPC_Vc z$OMqOz~*Ot$|axrNt60)6DMBr?o4$Vzf0pqPf_e9|9}MkETxxHqJ7+6NKV)r#Q6~e z1g=o%ko?B66VP#CQyDV(#mhhB$)A%S=aEBDN7gHm)ha=`g5r?B0k3N_+;cK6^s25wnoAGXu^}-Q0TaUXN@@Bk> zBq={6=3bbP6YPe1`kOAvoa0%HnUg9|iYPDp)Jq}t?c)>^-Y@lR@UD*+x5`0wA>;ih zNqH>~FSG^js? zDu}1^KIJ8!`tbwmQx8x)nXhyiZ+UrAuVL!qT`x)bjA(zDmV4~hD?;05DM9_26YA@~ z&Cx7g6stB~9rGl9FB4B-`@*YOlJe9NsQ-AVHl-EB*GO+}js zs6Pr5?F|yZS8Y{3x3Et>whXIYo|fNo9CnLxZ?|fNZq@5i_2&^1;48GM#pLtL(D1Kj z_)(=^=4#=j( zG|=zvCsI>TgSHIf1O=dYb6;KEBwZ8FfTAya!qLPOQK3K5kEXTHp02A4&4-{`(h2EP z|MYY*K2{y0W%$l@z#ke9ExX~PJ&nAO;7PGM{_Y6iTU&^qF1krNvG@TaCOz#~?_gcs z3h^7xkBtmWkHus4qYN(iWv1hVFT_km2ov+Dt(v1)I6RbzB?c1xEn{<;Oe#JwnH&~K znwEfnjr9RT!AkVSpMFC>KsT5He$uHnF&E2C3^OQ`<}a3~fIbUB+J_N0q|c(ak4>lA z6Bz>S?*l)pE`9V{`dI88fj)?}@Z#9WX`$74Vrinib75?xTZ_=r_}FA>IJPp#D7zVxVtwYC;SXIb5A$#6GaQ-V zd3GImD?V)Beu8gf{0zUbJ=!C4hP_k6iTcsuf$51jb1qz+8HubkPXZs~XMDm`#GlYa ze}AHLATg8}>YW=~7#m(aJ<^y#aN`p{PWUM)AfD)B>OX4vM`Vcjb=!|g^H*5NW33nT z>ADnsieW7bKTYJvhMN)aA-bL$X^gFn&&8S@c4d9jSKDdUA4WRAEPBr%=1S4)T+AXK z=gADtc#rlx%xRMRqYZrM@!J^z{Gt9<=r>L`P5nd%X8D?J!A_Oy1yEz85&C3PTl#BY zCt%aF(~5lDK=7@D4S;vrzkkT^A5H5c`tEpa%eBHDE_=l+dfI3U@(r^Z&u7y8LR8W( zVk9%ZqOmBccx(o~unsYf5k8lE-6@U`OTOXWS?oWvh4S?`2w!9=IbK*FlLX(i?g8J# z@x=NNMqv1P#=kfZd2^&%mwa}A)JQ%ydb}Ys5s#f7MZRQnh^S_!`w1WCQ$7Ea>qNv9 z!45@Z4-X|0u|#BHE;*dYjm26Qu-W0$c8Ks}bA>gS2)_@ z%r_$bB0yg%{vwNyvcCh3;NQ?^bfA@p3j47?boDMGKGu%`-W5+xJkS2q$?N%gh~OI{ zh4B3iLMtR&l!ajU34(7LBzRZ+%?tk|cDfdGn8!enq+9q1{fvxdQrUsYiFhVYwqNXn zzd84_aCpj)Pt(?j-iogOx!4Ts3x&D-r!&%Az>j9n2kWFi4)tVXqS>@(c3=%Yj93Or z&Yum_RCScN$6jL>=(M&t_~*7ra+?6gJp@98BZiR0`buKjRL=!;cL^WkGH zsHf6>@W)k_-^gd-$S_~Czw{A5Y{a&4g3mHMBQfE_LZ1xT-V6FG%rDh|A~f53W>Za_ zTu-cft+&+^ExexahnGf&8+eVWKXd<~)MMX&*!L~{bwpsqLj83^tWx@$SX#$cLY%wm zi2;!WMSQXT=8q%)#QPwxQ-4}SBpKFUl2H+STQ|Ya_JOrkwvVYlk?urP@}iUL@aIFh zLqN^^iuxEhqHUaSV-fOqM#AXX9LD~!nn4Y+G@++vt3GTSJp}u?^0j?GiXKP4D6$`M zX^ec69~}u#BHyizqTaLeP1`8R$NY=@%fp@se4$jn={TMTYb;Nk1v26>N+!dHwZKj_ z)r-`6EOH$3JNury6rQ}q?kX;zauRFD%+EZ8u=xxO|w7F6Fz3b z;z{>!t6pAUdXD%2j3Eys=>_XW>SaEa#Ieu8_o+^%GZod(Lf;v#qji0-QgWmZ`~F3u zMf&Ni!Axwtf3?jWhktQ^xz1k^I%w)vV&V|}#@t_{-NN-O`z(=s$7gX= z^WOhPXnu{6&=-3~VSJC%lvQui8Pp!{M%~$%AwS25>GRhl>zC~-;!Eg*1S+AA@ESO) z8zo$PNH^<=aAc^T_~g)Oqj@@*Xl?8tI7s6|M8fs7exeWdaP2Q`)0X`oruY~3!Wyth zQ?bMlT!j5MHJ4bQz@9VGG_Zg;#yFP89+8`9=ki}$_QcQf0uesTUBHiDi=?jQ@Y2{w zPyN`iz87MQ{3@XO)>~mOrZ4sztnVKe@kBV=*WoE4p=4fh3^y%&WAVgna=2kSg}Xg{ z(1J^TyZ;aMMZh0z_160FCfXO9v$&~^@p>AG{EJ^b{sW9%Oa_)uTcJmI~4RQ(zKrI6l%1< zi})+D3jIzIJS*9q@4E+quazLhhsg0J9>j|G z`JH+~*7?1M&L&G^4XrcD;R&H@=7qmTNk1%)E&q3~12;1h-uQg09bbcd>{)S^P`D-H zx?4CQOC|DgAN(8H4Z#Sa$F#_b5D|2l2~YiX4YGcP^6<@&1^cD3zfF&4JEzkL-Io}a zz9O2pe4E(cx>As@j)JD(JnPwyI=N2zn4|s`Gv&6QEx6Z7ADmB$^rNDL&t4C)mS~R< z$}3V&u2DWI)sKoqu*tg5zq-|l2+LDIpN-6%6rtZns>l8ZKSz#;73BMnDZietDYPTq zo7n#%bJ(Qn(Ab23vXMC+>B^q3XDJ_f=WAYydZ-?`kMhrS3UyJceV}Dzo$7m={PA_t z52r3$Jx`HQQvajNgE%iTf3EyIKTbr%2WJ``Kduo99M6eFv|AjdNPl$x(R2Pt<7$06 z6B4e+sOc-h{ndi_orNr=;(c`NwD$YfWdM;lhRT^f@2@U>hMIZGw0C%%@E6(} zX||NTV_l8#w-DiTo+qLwMgFFl@S}&YUt?gZknS4&V23}hb7~h<38*JR{Wf>JtZk+r zyt<5j@H#8Q9WP@4VEo6apMVkDiSE#+hrZnL?DoHbKHx9(zs=MSEv@kXQuZ8LrhXL| zv0M@tp7s)oJGdh1$pPSTd`FndG z`6K(Mh%e5^9cut$PB->D_7n}X{NBi=r2%9sd(>Ap$PTN&-Ujn zL)716{M0YPISqVJ+~Q-!cY^v`h4H-#pi=RDlKMX$=aI1eeBVp`FUC*xoPJ&u?rr%) z*DT3X;19H6l#1uYW#C`)vWKnz-nmZwC+v?R{-cQx)kRSLcPxfe>o*t)M$6970Ay4|mV=4OWOrMp&`V#vk7Dh!YxdSzd z<9P^g0<>Q*PYieFM0?s6AH9R5FPl9WE7hpQ?Gf(K;#25b;TL@ZV||pa@u8wcJUDVSotKy;Fd_Jg_A<iNqgXFgA(^J6wss2>eEqfkGb4~p_b{!I`6y#1T)C-lWb z@DnxO`XJtb8Z9{*na}$=ms5?anbBb`-GrCWbO+9~h5S)YE**d3;bign6G21h+C;rE zMf*DV%ZbC8HLu&3Tq!J`v$I#Qe7K4C*mxbgrrX~ve=cHQsXD0t%MOn$=Jqb*-*G>K i{}51f=;*Pd_4xOn;NOq*|K0zke}{kmxDWqv=ogV+l#-d3SX8Nyl#`#NP>@)elb@KPkXDqRt58r`keHd9pQm7?XP{?b$p8U| Y=H8b)57a0C_Nh~YM*GtY%Nlg~2DlSRPOGzwBu|Nm{rE&|3Qj3c-^Yh|!Qp-|vm>9&s z(#6Rcskw<=O3_Mrr8zlDI@L-gl?ACvR!T*w$@xVoN;*n;iMgr81&PTZ!Th3hy+j~8 zBUP^;F*7$mPY-NJdTL&3QDR9dOgXalw9M3;lwu{T=xVS@K&kjJ*Pviee?OqvP`fho zN|b7WR)eepDe(66bHObe;20F*8G>R!Qe{bMF^UmRq0Zi}A-D|*@egtIiHE2P#;wfF z)5jGqOWm-EkB1tLC4@jS@qUiJ2)mVvONuh{(oy^fRu$|Sg`_GcA5Be=e|WsJf2bc) zP++L>_4JGPc8!GlQwcLTeH|kRDFK=g9~|Nsf(T98q$xf-|mMTB7(o=Yq$N~{EiVP;8cE-1Z2Gn5iIxMOQ$b(B(4(-KQ_N|dZX zS*;d}5KGy`^*4lq@wXC^1+gg^@Wl1yM3Uy_(xpqrbSlamQ7N->fvQrdKJ^>Gbx zjmIonV3tACJbH>pGQt%Y{4Or8E?^JG1B-jt2zb!IO@S0c_*?=h0sTXRoDpRpTstU) z@o7fNV!+Jpg9vt*2?T>5pXtE(bPe(Y)-eH&As(21gBIubOmOoI3Jw8>GE%yNxs7lT zhX;8ABZ>MRLzoNCAS47GBwJv3JGK^79x?>{5SZz*cx~|AIfwtO-Pha1(!i~;RsSzT z#9mQ`w#Rku4C~Eg532ub`IU5cSE=CjQ+xlrZ&Ygu42>vVttcgHXZmgW?2e$#yY8Ma zyJ+zBZMI7>XFu7{WSuIqee6wHRe)aJ>AH zSSKZE68ZCg5gX&IazC5r`&0j>f8_XovYtWe*>m@2ZO42% z|Ln9mFhwL$T=THsa^aPm}Y9q+we z@a(yLu#us-m-y%ZbDqzTo_llV{1u&@1{Ytr$Q<@QDf%<|M1ZmKqNA;us}74Wc{8&} zcO4i0EPpv<`9!CuY0(>%CL1L+n8hDqn0QGr=wKStIdv6=t+mW-UdQqi4+L&xa68y| z-%0Z4|DU>j|795EXKzd0o2j<=*|s$)S7fe5Hk<6Z`+TeF!DKc4cOmnym$GEvp784Q z>RZ>8HysMe5!64oYTNv63O>QHxBL0m=qV}n)g_G|;-^yAkY4{@bh>{9#yxW~?%}Ww*Ges_$?x@kdJzOODR#=O1 zW`y^Ngbw4(6pziXxbyS8Kc&9f${V9n;u^PVM#1S4~KTce(K94@g`*>UuukPq9Z^mF%S>YXiq|01{f!^wj) zj=WN7)K$v7p(c2>+4+q1LX)3-tHf+3w#?sXZ@WY~UBTu*&%KB5#Cln)E$2+z@TGA3 zZygc3uNfajp%YYQh4-I1Q8j zGRo0!OQQV=At@^Xb8!OI(ac>Ly>0ZV-QEw593HINQMvrqbdA>SKB=w0yjbJ z!_$H(yCQwuGu5tHiBj1B06?P==VYBH-0zktP8cnGs>^Pnzw6>+~))v6l184~M bzlt@Ezs>O+$Jf@m&+lH`dB5?RpI7+>9WsiD literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 new file mode 100644 index 000000000..8118fd305 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_boolean","type":"BOOLEAN"},{"id":2,"name":"f_tinyint","type":"TINYINT"},{"id":3,"name":"f_smallint","type":"SMALLINT"},{"id":4,"name":"f_int","type":"INT"},{"id":5,"name":"f_bigint","type":"BIGINT"},{"id":6,"name":"f_float","type":"FLOAT"},{"id":7,"name":"f_double","type":"DOUBLE"},{"id":8,"name":"f_char","type":"CHAR(8)"},{"id":9,"name":"f_varchar","type":"VARCHAR(64)"},{"id":10,"name":"f_string","type":"STRING"},{"id":11,"name":"f_binary","type":"BINARY(8)"},{"id":12,"name":"f_varbinary","type":"VARBINARY(64)"},{"id":13,"name":"f_bytes","type":"VARBINARY(2147483647)"},{"id":14,"name":"f_blob","type":"BLOB"},{"id":15,"name":"f_blob_descriptor","type":"BLOB"},{"id":16,"name":"f_decimal_1_0","type":"DECIMAL(1, 0)"},{"id":17,"name":"f_decimal_9_2","type":"DECIMAL(9, 2)"},{"id":18,"name":"f_decimal_18_2","type":"DECIMAL(18, 2)"},{"id":19,"name":"f_decimal_19_2","type":"DECIMAL(19, 2)"},{"id":20,"name":"f_decimal_38_18","type":"DECIMAL(38, 18)"},{"id":21,"name":"f_decimal_38_38","type":"DECIMAL(38, 38)"},{"id":22,"name":"f_date","type":"DATE"},{"id":23,"name":"f_timestamp_0","type":"TIMESTAMP(0)"},{"id":24,"name":"f_timestamp_3","type":"TIMESTAMP(3)"},{"id":25,"name":"f_timestamp_6","type":"TIMESTAMP(6)"},{"id":26,"name":"f_timestamp_9","type":"TIMESTAMP(9)"},{"id":27,"name":"f_timestamp_ltz_0","type":"TIMESTAMP(0) WITH LOCAL TIME ZONE"},{"id":28,"name":"f_timestamp_ltz_3","type":"TIMESTAMP(3) WITH LOCAL TIME ZONE"},{"id":29,"name":"f_timestamp_ltz_6","type":"TIMESTAMP(6) WITH LOCAL TIME ZONE"},{"id":30,"name":"f_timestamp_ltz_9","type":"TIMESTAMP(9) WITH LOCAL TIME ZONE"},{"id":31,"name":"f_variant","type":"VARIANT"},{"id":32,"name":"f_array_int","type":{"type":"ARRAY","element":"INT"}},{"id":33,"name":"f_map_string_bigint","type":{"type":"MAP","key":"STRING NOT NULL","value":"BIGINT"}},{"id":34,"name":"f_row","type":{"type":"ROW","fields":[{"id":35,"name":"nested_int","type":"INT"},{"id":36,"name":"nested_string","type":"STRING"},{"id":37,"name":"nested_decimal","type":"DECIMAL(19, 4)"},{"id":38,"name":"nested_timestamp","type":"TIMESTAMP(9)"},{"id":39,"name":"nested_timestamp_ltz","type":"TIMESTAMP(9) WITH LOCAL TIME ZONE"}]}},{"id":40,"name":"f_array_array_int","type":{"type":"ARRAY","element":{"type":"ARRAY","element":"INT"}}},{"id":41,"name":"f_array_map","type":{"type":"ARRAY","element":{"type":"MAP","key":"STRING NOT NULL","value":"INT"}}},{"id":42,"name":"f_map_array","type":{"type":"MAP","key":"STRING NOT NULL","value":{"type":"ARRAY","element":"INT"}}},{"id":43,"name":"f_array_row","type":{"type":"ARRAY","element":{"type":"ROW","fields":[{"id":44,"name":"name","type":"STRING"},{"id":45,"name":"score","type":"DECIMAL(9, 2)"}]}}},{"id":46,"name":"f_map_row","type":{"type":"MAP","key":"STRING NOT NULL","value":{"type":"ROW","fields":[{"id":47,"name":"enabled","type":"BOOLEAN"},{"id":48,"name":"event_time","type":"TIMESTAMP(6)"}]}}},{"id":49,"name":"f_deep_row","type":{"type":"ROW","fields":[{"id":50,"name":"children","type":{"type":"ARRAY","element":{"type":"ROW","fields":[{"id":51,"name":"leaf_id","type":"BIGINT"},{"id":52,"name":"leaf_variant","type":"VARIANT"}]}}},{"id":53,"name":"labels","type":{"type":"MAP","key":"STRING NOT NULL","value":"STRING"}}]}},{"id":54,"name":"f_array_variant","type":{"type":"ARRAY","element":"VARIANT"}},{"id":55,"name":"f_map_variant","type":{"type":"MAP","key":"STRING NOT NULL","value":"VARIANT"}}],"highestFieldId":55,"partitionKeys":[],"primaryKeys":[],"options":{"manifest.format":"avro","blob-field":"f_blob,f_blob_descriptor","blob-descriptor-field":"f_blob_descriptor","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1","row-tracking.enabled":"true"},"comment":"Paimon type compatibility data written by Rust Paimon","timeMillis":1788858154980} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 new file mode 100644 index 000000000..93ff344a5 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-0","deltaManifestList":"manifest-list-3bf7fb3d-4f1e-4d6d-9ecc-a90c18cf7151-1","commitUser":"12acc040-0373-49d8-9406-4dcc86ea02cf","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155006,"totalRecordCount":6,"deltaRecordCount":6,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/bucket-0/data-38bc7437-0dec-4de2-be1a-44561281c6db-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ad2eefb5d2ef434595571d5ea7455a6767b81b28 GIT binary patch literal 4417 zcmb_g&rcgy5Pn{790P_xn%&sQs!%JeR7tZW1ZqcAVb|D(1PFG(4!#60nBcOusSSqY zL=&k>rKpP9`~f-iP^lbxXsf8|fnKT}st8p*j=?vu1}RI^;Tzh~@k7@$ygWrw*44}iTPT|m@M@NJ=X zeN>BTx8M6g*az9Y%aMNr$ep8%J#J&{(XsyndG6L8#r)_)8yjx7vuB@*Y_r?(iZRbg zg_>;ieE{k9{`tjAGb5N4_PlKx^T&QX`8j5VZN0e*xnS&pGSvc~|DGkt$tA{mwi#O) zX6zn3U&8Zr_|=f-q`6uy@7mZ?M&pl5OxR|w;muAa0Wyy6VUeA_JJWG@P+WnI``3l` z92Y!p5t|Y|)R%%sh6RdkAhX#d6nK21(-Q_Q7vB=G$H0Qxs;L6D3Mf!_G*!Tgg8~(~ znS#?QoX71f*QS{w#L%-1NR;Ac3QkNAG&v}l)9&F0R$e(QT?FRLCQ(a026B?0=fS8yLq8|iF1w_0nrn#tYD*_2^e2#$jhdH|u*L~xZ6EMw<^#_07}*+$RYw9Hx? z%*3jw(X69vwrJ#2oU5zB%2{Vsakh|O(~UHxsH4HGwc1P;Jzmd!AM---2AHGMgQb#O zvM5i>8@{m5>E$Pd9~+sSqI1JnU0uwBcRG|9FOp99F!_0hbVB~6RXWQMlzDoY!{cwL zI@Q^#FSv8bL-M3=(>657PT6d57!e!H347?R%7J9^iw;Sa@3cvSw|T)=a=Knn(`~XBWfy zqyv&6?>J@A=aheHJ%Lxe+oi^0R}&ZcmF(W>P;m4|!>X!A;lXi5RoB(vP4zO0;H#<{ z$zcg=RUN|>h^#NHP-!HJY8t|X)<;w|xJ+VH3$Mc#tZ5%!!Gd2=07HNL2H=KV*bI!EXrY!Hhk;( zUI1AQ->e=9EG?i3rH)>Gn9l~j=;|=OqlEzQ%_AHH1EW&E7_BKTDq6DCZ`Mz{sVrLc zv-pI9*m&y1H0F;i>BY-CigU=bp(o;N5c?FZjllV2VzN)~zq~k`HF7DcFZrX_xhd=y z!YBX4cLeHeGM@>Bug%2c;rPf(HkF7Mli|Qh5_eTKrIZvUg)^ha#)hFo ztN8hJ>K#SdAfZ(hO|E(~xgJL2;)<75dsl4Qtwr5st^=(ZOWafWD6{rV>1fYbA*9Jk^KdN5?3MT7@qN&9<%YKZ365zuS5uT+|~Lw@SSth}Tdre%{f+ z{`Jz*cy;L%#Ub?y74Uu_>M`i~=Jg8Zkv~5V2+Gq_UoVnY*Q-CaqUQ{Fc)ly=E9kG_ zQU0l(6BsJ5?0)yjHS9k)7x10H^A2MjO|IdLm~tWSZ$Jx$NlK`2Y104*=$B}9E5jms z(0xox7w{I3`Pk8k@%a*Y_`-8p`K>{i}`P{X%^g zTIZKL6fvZN90v72?BmsKjUWXHSt32P$hRu{lq3E-q49H1ET}#J3cq{Amk89yOG*dE x3sEq)XC+m5e=WWGcA?l)N*9Wme6DA}fA+%JKF@tG{8irUAM0ypYzqGKljA6n5PL9S}mOsyd>ZIwT=!Lt8r4O|Gg{$0?4}2CC?|*w@s^v4i7QDp8~g zRY+t&l@Jnt0*L{B06Q}iLx&;;1|~K@obAMR<0dImyRg}J@4olmci;Q$eC6@T)k(Z( znTdw7Lu$AUwt%s#>mb$y_I{m!MPL!dv{VoS2Ck7t9V^bFX;l-LKJSo39cwkyNVp(X zVh{`4q#GVSzO4~mZGgqCri(gBDHgq;dk*crn|5cb$`DUOoj- z*|$mK6vT2dvj*k>!cL1-U`Pwlt%9sz*1f zz&{K{>1I!il;1>|d{GQh5sAt1B3gqF{60aH&Ky@77D5qFLFVLu&CAi0#~X6W@$;QQ zq0`g`Clp~&PoKVqM~Ov2@K_AMBF$mT!uym%%_cQR@121HF5Tr;IR;d+jdyh$EIOms zI%6p`0&Y=$f8LyQAumIR$Ro?o`2R8L2%-S*7onJ;vsKJ3Ln%;vbS9|a&mUdLtil|J z_>gNw5Cymag&`cC>hYSU>l(Gvh+IS3%t8Um5Jjz}o2;YxUd4wXAf-Q>g;}>@MC~4yd}@3Y z&xQt7;yGZxC}jc<^p86X4>}wgF*@8rKzkQqDE))b>3u>;q~DlzPF{`n*0J+J2rIl) zkX@ETy6V<8oZ?N1r-dIH1F1H*%oN)gp6jLS2t*2CvhKen9Ay`Z( zX02|KMnf|Vq?0|OvkddS`K!r&Nxr`K^2hkwpYK1jZFc&@o!?C9>nL-0g&kw2n2~K2 z+c-YEP*LX7v-9yJO%PA3WF}rADI8Cy@6M%W7E;QbT8Ss=pN;iom)OtX7(2zV4Ett; zVVH5|56j-9Z|#Hj;n5CzgCWz`pC7$?al1TmGM-(U7=5?&?f5wPDE$eiK3+RHnCz|V EFUaZ=QUCw| literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/manifest/manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0 new file mode 100644 index 0000000000000000000000000000000000000000..018d7237f72106177bd607f6cad45fe2bdffb31a GIT binary patch literal 1117 zcmcIk%SyvQ6s<0FCnDkpjN6cr(`4F4l1Z3M3$=9TLi`H%euiJ* zKllx>c9~OJqL@h=25uaZCfuf|H_{`2!N;=fgIj z!zcpM1n_Z6Km#!1CoBLGpg1O3iv7Hpu)d7Nc|hb8hjBvXf}~F<NSbq1{yn^l=G*``cXi2idpBvx0SLo`n1@-M@w`~=wwsljDWbXDmY2N2U& z7xz(e7)7XYjQC^>sErtSDf3YSB{YU?OugbTWh0s9`vZ{2G>p?kE=fj^0HYlK!0Y8f z2$7$m_Wp6nlwXlP?%8_R%DGRq!i1w~E?r0!fcv_EiKUvjf@GY9$!HN}>yK-wD4oMt zu1Qo)!&=E{SA0v68aA+V4(vm<=jcvuf!L1fRIL9eL_!wzmZWOx7U`>oi>vpb;e zS7@fXcMg>08V2cfUF$4!lrcKV6Ad}UMT1W#2$j(erh^9ZA)tp5N9|&^rb~`$J-0i% z*i?zGozbJh;*Q&SfSnm=rdrIiW|2jMK!Y%oo01efyEKR{O`x6Y<@L z&tJs%Yd21A{uK5GqVRoPyznHZNfNz6eVc+ fL32yG_3gSSh*tzrDAv00>GjiVZ#G_t^D@5x%A<=| literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 new file mode 100644 index 000000000..3ee403b98 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_vector_boolean","type":{"type":"VECTOR","element":"BOOLEAN","length":3}},{"id":2,"name":"f_vector_tinyint","type":{"type":"VECTOR","element":"TINYINT","length":3}},{"id":3,"name":"f_vector_smallint","type":{"type":"VECTOR","element":"SMALLINT","length":3}},{"id":4,"name":"f_vector_int","type":{"type":"VECTOR","element":"INT","length":3}},{"id":5,"name":"f_vector_bigint","type":{"type":"VECTOR","element":"BIGINT","length":3}},{"id":6,"name":"f_vector_float","type":{"type":"VECTOR","element":"FLOAT","length":3}},{"id":7,"name":"f_vector_double","type":{"type":"VECTOR","element":"DOUBLE","length":3}}],"highestFieldId":7,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","bucket":"-1"},"comment":null,"timeMillis":1788858155008} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 new file mode 100644 index 000000000..e0535a0c0 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_vector_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-0","deltaManifestList":"manifest-list-5c419dc6-cdb8-4535-bc12-4d1c12a76a6c-1","commitUser":"8b326938-c073-4993-bd3e-4761ca4e3e0d","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155020,"totalRecordCount":2,"deltaRecordCount":2} \ No newline at end of file From 132cba6e5a23d04e4a9db30768337f56d25f5475 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Wed, 9 Sep 2026 10:38:38 +0800 Subject: [PATCH 2/3] test(parquet): clarify nested blob schema error --- test/inte/paimon_read_compat_inte_test.cpp | 3 +-- test/test_data/parquet/append_types_compatibility.db/README.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index df6d86a52..f5b58ad9e 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -475,8 +475,7 @@ TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError) { std::vector UnsupportedReadParams() { const std::vector read_cases = { - {"ArrayBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, - {"MapBlob", "nested_blob_types", "f_map_blob", "Blob field must be a top-level field"}, + {"NestedBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md b/test/test_data/parquet/append_types_compatibility.db/README.md index db2486549..5e4e330be 100644 --- a/test/test_data/parquet/append_types_compatibility.db/README.md +++ b/test/test_data/parquet/append_types_compatibility.db/README.md @@ -33,7 +33,7 @@ Each writer prefix has four tables with the same schema and logical values: - `_types`: 43 columns and 3 rows covering integral and floating types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE, TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions, VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in every nullable column and row 3 exercises empty values. - `_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, and DOUBLE. -- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ currently rejects these because it only permits BLOB as a top-level field. +- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ rejects the table schema because ARRAY<BLOB> is unsupported. Schema validation happens before column projection, so the compatibility test explicitly checks this error instead of reading either field. - `_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++ currently rejects `TIME` while parsing the table schema. The tables are separated because Paimon C++ does not allow VECTOR in a data-evolution table, BLOB requires data evolution, and a schema-level incompatibility must not prevent compatible columns from being tested. From ac1d2ac7d476fbcedf3bf30cc5651d1191471937 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Wed, 9 Sep 2026 15:22:52 +0800 Subject: [PATCH 3/3] fix(parquet): make blob casting idempotent --- .../casting/binary_to_blob_cast_executor.cpp | 15 +- .../core/casting/cast_executor_test.cpp | 12 ++ src/paimon/core/io/field_mapping_reader.cpp | 26 +--- test/inte/paimon_read_compat_inte_test.cpp | 129 ++++++++++-------- 4 files changed, 97 insertions(+), 85 deletions(-) diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp index 77ee39e80..bbe2f7b7a 100644 --- a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp @@ -45,16 +45,21 @@ Result BinaryToBlobCastExecutor::Cast( Result> BinaryToBlobCastExecutor::Cast( const std::shared_ptr& array, const std::shared_ptr& target_type, arrow::MemoryPool* pool) const { - if (array->type_id() != arrow::Type::BINARY) { - return Status::Invalid( - fmt::format("BinaryToBlobCastExecutor only supports binary input, got {}", - array->type()->ToString())); - } if (target_type->id() != arrow::Type::LARGE_BINARY) { return Status::Invalid( fmt::format("BinaryToBlobCastExecutor only supports large_binary target, got {}", target_type->ToString())); } + // Parquet may have already restored the logical BLOB representation from ARROW:schema. + if (array->type_id() == arrow::Type::LARGE_BINARY) { + return array; + } + if (array->type_id() != arrow::Type::BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports binary or large_binary input, " + "got {}", + array->type()->ToString())); + } auto binary_array = checked_pointer_cast(array); if (binary_array->offset() != 0) { diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index 3905ba04d..9035eb295 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -1427,6 +1427,18 @@ TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArray) { ASSERT_EQ(target_array->data()->buffers[2], src_array->data()->buffers[2]); } +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorAlreadyLargeBinary) { + auto cast_executor = std::make_shared(); + auto src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::large_binary(), R"(["foo", null, "bar"])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr target_array, + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool())); + ASSERT_EQ(target_array, src_array); +} + TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArrayWithOffset) { auto cast_executor = std::make_shared(); auto src_array = diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index ca8a28be8..e2c8b4cfd 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -31,12 +31,10 @@ #include "arrow/scalar.h" #include "fmt/format.h" #include "paimon/common/data/binary_string.h" -#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" -#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" #include "paimon/core/utils/field_mapping.h" @@ -144,22 +142,11 @@ Result> FieldMappingReader::CastNonPartitionArrayI CastingUtils::Cast(dict_array, /*target_type=*/arrow::utf8(), arrow::compute::CastOptions::Safe(), arrow_pool_.get())); } - const DataField& read_field = non_partition_info_.non_partition_read_schema[i]; - // The Parquet reader may already have restored an inline BLOB through ARROW:schema. - // Only bypass the physical binary-to-BLOB cast when both the configured executor and - // the logical field metadata confirm that this is a BLOB conversion. - const bool is_already_logical_blob = - std::dynamic_pointer_cast( - non_partition_info_.cast_executors[i]) != nullptr && - single_column_array->type_id() == arrow::Type::LARGE_BINARY && - BlobUtils::IsBlobField(read_field.ArrowField()); - if (is_already_logical_blob) { - column = single_column_array; - } else { - PAIMON_ASSIGN_OR_RAISE( - column, non_partition_info_.cast_executors[i]->Cast( - single_column_array, read_field.Type(), arrow_pool_.get())); - } + PAIMON_ASSIGN_OR_RAISE( + column, + non_partition_info_.cast_executors[i]->Cast( + single_column_array, non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); } else { // read and data type may both be string type, but after adapter transform, type may be // dictionary, need reconstruct struct type @@ -169,8 +156,7 @@ Result> FieldMappingReader::CastNonPartitionArrayI // read types differ -- the reader may hand back a dictionary-encoded array // for an unchanged type, which is not a reshape target. if (!non_partition_info_.non_partition_data_schema[i].Type()->Equals( - *non_partition_info_.non_partition_read_schema[i].Type()) && - !column->type()->Equals(*non_partition_info_.non_partition_read_schema[i].Type())) { + *non_partition_info_.non_partition_read_schema[i].Type())) { PAIMON_ASSIGN_OR_RAISE( column, NestedProjectionUtils::AlignArrayToReadType( column, non_partition_info_.non_partition_read_schema[i].Type(), diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index f5b58ad9e..94d6c99fc 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -45,11 +45,6 @@ namespace paimon::test { namespace { -const std::map kBlobDescriptorReadOptions = { - {Options::FILE_SYSTEM, "local"}, - {Options::BLOB_AS_DESCRIPTOR, "true"}, - {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, -}; const char kDatabaseName[] = "append_types_compatibility"; std::vector WriterPrefixes() { @@ -145,7 +140,7 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { const std::string& writer_prefix = GetParam(); TimezoneGuard timezone_guard("Asia/Shanghai"); - // Project every field from the main compatibility table in schema order. + // Project every non-BLOB field from the main compatibility table in schema order. std::vector fields = { // Basic numeric types. "id", @@ -163,9 +158,6 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { "f_binary", "f_varbinary", "f_bytes", - // BLOB values stored as external or inline descriptors. - "f_blob", - "f_blob_descriptor", // DECIMAL values across physical precision boundaries. "f_decimal_1_0", "f_decimal_9_2", @@ -198,7 +190,7 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { "f_map_variant", }; ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadTable(writer_prefix + "_types", fields, kBlobDescriptorReadOptions)); + ReadTable(writer_prefix + "_types", fields)); std::shared_ptr rows = GetOnlyStructChunk(result); ASSERT_TRUE(rows); ASSERT_EQ(rows->length(), 3); @@ -224,54 +216,6 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { R"(["varbinary\u0000value", null, ""])"); AssertFieldEqualsJson(rows, "f_bytes", arrow::binary(), R"(["bytes\u0000value", null, ""])"); - // External BLOB descriptors, including a non-empty and an empty blob range. - std::shared_ptr blobs = - std::dynamic_pointer_cast(rows->GetFieldByName("f_blob")); - ASSERT_TRUE(blobs); - ASSERT_EQ(blobs->length(), 3); - ASSERT_TRUE(blobs->IsNull(1)); - std::string_view first = blobs->GetView(0); - ASSERT_OK_AND_ASSIGN(std::unique_ptr first_descriptor, - BlobDescriptor::Deserialize(first.data(), first.size())); - ASSERT_EQ(first_descriptor->Version(), 2); - ASSERT_EQ(first_descriptor->Offset(), 4); - ASSERT_EQ(first_descriptor->Length(), 41); - ASSERT_EQ(first_descriptor->Uri().substr(first_descriptor->Uri().size() - 5), ".blob"); - std::string_view empty = blobs->GetView(2); - ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_descriptor, - BlobDescriptor::Deserialize(empty.data(), empty.size())); - ASSERT_EQ(empty_descriptor->Offset(), 61); - ASSERT_EQ(empty_descriptor->Length(), 0); - - // Resolve the external BLOB and validate its payload instead of its descriptor. - std::map blob_value_options = { - {Options::FILE_SYSTEM, "local"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, - }; - ASSERT_OK_AND_ASSIGN(std::shared_ptr blob_value_result, - ReadTable(writer_prefix + "_types", {"f_blob"}, blob_value_options)); - std::shared_ptr blob_value_rows = GetOnlyStructChunk(blob_value_result); - ASSERT_TRUE(blob_value_rows); - AssertFieldEqualsJson(blob_value_rows, "f_blob", arrow::large_binary(), - R"(["ordinary blob payload from pypaimon 2.0.0", null, ""])"); - - // Inline BLOB descriptors exercise compatible binary types restored from ARROW:schema. - std::shared_ptr inline_blobs = - std::dynamic_pointer_cast( - rows->GetFieldByName("f_blob_descriptor")); - ASSERT_TRUE(inline_blobs); - ASSERT_EQ(inline_blobs->length(), 3); - ASSERT_FALSE(inline_blobs->IsNull(0)); - ASSERT_TRUE(inline_blobs->IsNull(1)); - ASSERT_TRUE(inline_blobs->IsNull(2)); - std::string_view inline_blob = inline_blobs->GetView(0); - ASSERT_OK_AND_ASSIGN(std::unique_ptr inline_descriptor, - BlobDescriptor::Deserialize(inline_blob.data(), inline_blob.size())); - ASSERT_EQ(inline_descriptor->Version(), 2); - ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); - ASSERT_EQ(inline_descriptor->Offset(), 7); - ASSERT_EQ(inline_descriptor->Length(), 11); - // DECIMAL values around the int32, int64, and fixed-byte-array precision boundaries. AssertFieldEqualsJson(rows, "f_decimal_1_0", arrow::decimal128(1, 0), R"(["9", null, null])"); AssertFieldEqualsJson(rows, "f_decimal_9_2", arrow::decimal128(9, 2), @@ -413,9 +357,74 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { ASSERT_TRUE(map_variant_values); AssertVariantJsonAt(map_variant_values, 0, object_variant_json); AssertVariantJsonAt(map_variant_values, 1, array_variant_json); +} + +TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) { + const std::string& writer_prefix = GetParam(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadTable(writer_prefix + "_types", {"f_blob", "f_blob_descriptor"}, + {{Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + std::shared_ptr rows = GetOnlyStructChunk(result); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 3); + ASSERT_EQ(rows->num_fields(), 3); + + // External BLOB descriptors, including a non-empty and an empty blob range. + std::shared_ptr blobs = + std::dynamic_pointer_cast(rows->GetFieldByName("f_blob")); + ASSERT_TRUE(blobs); + ASSERT_EQ(blobs->length(), 3); + ASSERT_TRUE(blobs->IsNull(1)); + std::string_view first = blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_descriptor, + BlobDescriptor::Deserialize(first.data(), first.size())); + ASSERT_EQ(first_descriptor->Version(), 2); + ASSERT_EQ(first_descriptor->Offset(), 4); + ASSERT_EQ(first_descriptor->Length(), 41); + ASSERT_EQ(first_descriptor->Uri().substr(first_descriptor->Uri().size() - 5), ".blob"); + std::string_view empty = blobs->GetView(2); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_descriptor, + BlobDescriptor::Deserialize(empty.data(), empty.size())); + ASSERT_EQ(empty_descriptor->Offset(), 61); + ASSERT_EQ(empty_descriptor->Length(), 0); + + // Resolve the external BLOB and validate its payload instead of its descriptor. + std::map blob_value_options = { + {Options::FILE_SYSTEM, "local"}, + {Options::BLOB_AS_DESCRIPTOR, "false"}, + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob_value_result, + ReadTable(writer_prefix + "_types", {"f_blob"}, blob_value_options)); + std::shared_ptr blob_value_rows = GetOnlyStructChunk(blob_value_result); + ASSERT_TRUE(blob_value_rows); + AssertFieldEqualsJson(blob_value_rows, "f_blob", arrow::large_binary(), + R"(["ordinary blob payload from pypaimon 2.0.0", null, ""])"); + + // Inline BLOB descriptors exercise compatible binary types restored from ARROW:schema. + std::shared_ptr inline_blobs = + std::dynamic_pointer_cast( + rows->GetFieldByName("f_blob_descriptor")); + ASSERT_TRUE(inline_blobs); + ASSERT_EQ(inline_blobs->length(), 3); + ASSERT_FALSE(inline_blobs->IsNull(0)); + ASSERT_TRUE(inline_blobs->IsNull(1)); + ASSERT_TRUE(inline_blobs->IsNull(2)); + std::string_view inline_blob = inline_blobs->GetView(0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr inline_descriptor, + BlobDescriptor::Deserialize(inline_blob.data(), inline_blob.size())); + ASSERT_EQ(inline_descriptor->Version(), 2); + ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); + ASSERT_EQ(inline_descriptor->Offset(), 7); + ASSERT_EQ(inline_descriptor->Length(), 11); +} + +TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) { + const std::string& writer_prefix = GetParam(); - // VECTOR values are stored separately because VECTOR and BLOB require incompatible - // table options. Validate every supported VECTOR element type here. + // Validate every supported VECTOR element type. std::vector vector_fields = { "id", "f_vector_boolean", "f_vector_tinyint", "f_vector_smallint", "f_vector_int", "f_vector_bigint", "f_vector_float", "f_vector_double",