From 7d2cffc061c3db961bf3981fdb5d5d7f3361d322 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Thu, 10 Sep 2026 19:36:33 +0800 Subject: [PATCH 1/2] refactor(parquet): introduce reusable read type adapter --- .../core/schema/arrow_schema_validator.cpp | 8 +- .../schema/arrow_schema_validator_test.cpp | 17 ++- src/paimon/core/schema/schema_validation.cpp | 6 +- .../core/schema/schema_validation_test.cpp | 3 +- .../core/table/format/format_table_read.cpp | 2 +- src/paimon/format/parquet/CMakeLists.txt | 4 +- .../parquet/parquet_file_batch_reader.cpp | 39 ++---- .../parquet/parquet_file_batch_reader.h | 7 +- ...rter.cpp => parquet_read_type_adapter.cpp} | 111 +++++++++++------- ...onverter.h => parquet_read_type_adapter.h} | 24 +++- ...cpp => parquet_read_type_adapter_test.cpp} | 69 ++++++----- test/inte/paimon_read_compat_inte_test.cpp | 26 +++- .../append_types_compatibility.db/README.md | 18 ++- ...58f-c0a7-468e-8d71-45b65142d422-0.parquet} | Bin ...49858f-c0a7-468e-8d71-45b65142d422-1.blob} | Bin ...st-4db45a0c-f933-4043-80de-f1d5a5107966-0} | Bin 2249 -> 2232 bytes ...st-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0} | Bin 1006 -> 1006 bytes ...ist-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 | Bin 0 -> 1115 bytes .../java_array_blob_types/schema/schema-0 | 28 +++++ .../snapshot/EARLIEST | 0 .../snapshot/LATEST | 0 .../java_array_blob_types/snapshot/snapshot-1 | 18 +++ ...d467-f565-4453-b400-468f9ec22a54-0.parquet | Bin 0 -> 298 bytes ...b0d467-f565-4453-b400-468f9ec22a54-1.blob} | Bin ...est-62c45eba-0a54-4a63-b89d-0ab09a768412-0 | Bin 0 -> 2233 bytes ...st-d3262408-2da8-408b-9c67-b487ecf8565b-0} | Bin 1111 -> 1006 bytes ...ist-d3262408-2da8-408b-9c67-b487ecf8565b-1 | Bin 0 -> 1115 bytes .../schema/schema-0 | 13 +- .../snapshot/EARLIEST} | 0 .../snapshot/LATEST | 0 .../java_map_blob_types/snapshot/snapshot-1 | 18 +++ .../snapshot/snapshot-1 | 18 --- ...3fd246-38df-40f7-9147-c1c416282ecc-0.blob} | Bin ...b1a-5db9-478e-a727-6f34d6e800e5-0.parquet} | Bin ...st-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0} | Bin 2333 -> 2296 bytes ...st-2f17c472-fea6-42d0-8300-12da4874d53a-0} | Bin 799 -> 799 bytes ...st-2f17c472-fea6-42d0-8300-12da4874d53a-1} | Bin 899 -> 899 bytes .../python_array_blob_types/schema/schema-0 | 33 ++++++ .../python_array_blob_types/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 15 +++ ...b349-773b-4e2f-b364-ad98ee38fa72-0.parquet | Bin 0 -> 457 bytes ...33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob} | Bin ...est-59ba3981-44aa-4af6-960e-9048ce6e883b-0 | Bin 0 -> 2298 bytes ...ist-c762398d-e22c-4676-b43d-4564909a0859-0 | Bin 0 -> 799 bytes ...ist-c762398d-e22c-4676-b43d-4564909a0859-1 | Bin 0 -> 899 bytes .../schema/schema-0 | 17 +-- .../python_map_blob_types/snapshot/LATEST | 1 + .../python_map_blob_types/snapshot/snapshot-1 | 15 +++ .../snapshot/snapshot-1 | 15 --- ...3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet | Bin 0 -> 1124 bytes ...est-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 | Bin 0 -> 2421 bytes ...ist-edec5851-5eca-466b-9bde-780df761f932-0 | Bin 0 -> 1117 bytes ...st-edec5851-5eca-466b-9bde-780df761f932-1} | Bin 1117 -> 1219 bytes .../rust_array_blob_types/schema/schema-0 | 1 + .../rust_array_blob_types/snapshot/LATEST | 1 + .../rust_array_blob_types/snapshot/snapshot-1 | 1 + ...7a11-39ca-4131-b12d-81f824f57978-0.parquet | Bin 0 -> 1503 bytes ...est-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 | Bin 0 -> 2418 bytes ...ist-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 | Bin 0 -> 1117 bytes ...st-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1} | Bin 1219 -> 1219 bytes .../rust_map_blob_types/schema/schema-0 | 1 + .../rust_map_blob_types/snapshot/LATEST | 1 + .../rust_map_blob_types/snapshot/snapshot-1 | 1 + ...ab27-cf5e-48e5-8449-3f674480e077-0.parquet | Bin 2015 -> 0 bytes ...est-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 | Bin 2432 -> 0 bytes .../rust_nested_blob_types/schema/schema-0 | 1 - .../snapshot/snapshot-1 | 1 - 67 files changed, 348 insertions(+), 186 deletions(-) rename src/paimon/format/parquet/{parquet_timestamp_converter.cpp => parquet_read_type_adapter.cpp} (69%) rename src/paimon/format/parquet/{parquet_timestamp_converter.h => parquet_read_type_adapter.h} (59%) rename src/paimon/format/parquet/{parquet_timestamp_converter_test.cpp => parquet_read_type_adapter_test.cpp} (76%) rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet => java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet} (100%) rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob => java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob} (100%) rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 => java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0} (88%) rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 => java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0} (95%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types => java_array_blob_types}/snapshot/EARLIEST (100%) rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types => java_array_blob_types}/snapshot/LATEST (100%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob => java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob} (100%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 => java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0} (86%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1 rename test/test_data/parquet/append_types_compatibility.db/{java_nested_blob_types => java_map_blob_types}/schema/schema-0 (69%) rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/snapshot/LATEST => java_map_blob_types/snapshot/EARLIEST} (100%) rename test/test_data/parquet/append_types_compatibility.db/{rust_nested_blob_types => java_map_blob_types}/snapshot/LATEST (100%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1 delete mode 100644 test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob => python_array_blob_types/bucket-0/data-043fd246-38df-40f7-9147-c1c416282ecc-0.blob} (100%) rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet => python_array_blob_types/bucket-0/data-3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet} (100%) rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 => python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0} (87%) rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 => python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0} (96%) rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 => python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1} (85%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob => python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob} (100%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1 rename test/test_data/parquet/append_types_compatibility.db/{python_nested_blob_types => python_map_blob_types}/schema/schema-0 (61%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 delete mode 100644 test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 rename test/test_data/parquet/append_types_compatibility.db/{rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 => rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1} (89%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 rename test/test_data/parquet/append_types_compatibility.db/{rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 => rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1} (89%) create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0 create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST create mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1 delete mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet delete mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 delete mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 delete mode 100644 test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 009979365..c008d222d 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -173,7 +173,9 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( case arrow::Type::type::LARGE_BINARY: { if (BlobUtils::IsBlobMetadata(key_value_metadata)) { if (!allow_blob) { - return Status::Invalid("Blob field must be a top-level field."); + return Status::Invalid( + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } break; } @@ -258,7 +260,9 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::LARGE_BINARY: { if (BlobUtils::IsBlobField(field)) { if (!allow_blob) { - return Status::Invalid("Blob field must be a top-level field."); + return Status::Invalid( + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } break; } diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index d644e5635..283b61ec2 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -170,7 +170,7 @@ TEST(ArrowSchemaValidatorTest, TestInvalidDataType) { } } -TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { +TEST(ArrowSchemaValidatorTest, TestBlobFieldPlacement) { { auto arrow_schema = arrow::schema(arrow::FieldVector({BlobUtils::ToArrowField("blob", true)})); @@ -186,14 +186,16 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::field("nested", arrow::struct_({BlobUtils::ToArrowField("blob", true)})); auto arrow_schema = arrow::schema(arrow::FieldVector({nested_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto array_blob_field = arrow::field("array_blob", arrow::list(BlobUtils::ToArrowField("item", true))); auto arrow_schema = arrow::schema(arrow::FieldVector({array_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto map_blob_field = arrow::field( @@ -211,7 +213,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::map(arrow::utf8(), arrow::struct_({BlobUtils::ToArrowField("blob", true)}))); auto arrow_schema = arrow::schema(arrow::FieldVector({map_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto nested_map_blob_field = arrow::field( @@ -220,7 +223,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", true)))); auto arrow_schema = arrow::schema(arrow::FieldVector({nested_map_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { std::vector nested_fields = { @@ -230,7 +234,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::field("nested", DataField::ConvertDataFieldsToArrowStructType(nested_fields)))}; auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 19fbdb136..0dd2533d3 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -789,9 +789,9 @@ Status SchemaValidation::ValidateMosaicDataFields(const TableSchema& schema, const std::vector inline_blob_fields = options.GetBlobInlineFields(); const std::set inline_blob_field_set(inline_blob_fields.begin(), inline_blob_fields.end()); - // Match Java SchemaValidation by validating only fields stored in the normal data file. C++ - // permits BLOB only as a top-level field; descriptor and view fields are inline, so Mosaic - // must reject them here. + // Match Java SchemaValidation by validating only fields stored in the normal data file. + // Top-level BLOB fields stored in separate files are skipped; descriptor and view fields are + // inline, so Mosaic must reject them here. for (const DataField& field : schema.Fields()) { if (BlobUtils::IsBlobField(field.ArrowField()) && inline_blob_field_set.count(field.Name()) == 0) { diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index fed7ea5a5..117555782 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -1245,7 +1245,8 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { }); ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, nested_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } TEST(SchemaValidationTest, TestMapSharedShreddingCompression) { diff --git a/src/paimon/core/table/format/format_table_read.cpp b/src/paimon/core/table/format/format_table_read.cpp index 7bc14b59f..99dda9787 100644 --- a/src/paimon/core/table/format/format_table_read.cpp +++ b/src/paimon/core/table/format/format_table_read.cpp @@ -60,7 +60,7 @@ namespace { /// `table_unit`. A file's unit means something else in each format, so the rule does too: /// /// - parquet records the unit, so the file's is the one its values are in. Only `MILLI` under a -/// `SECOND` column is rescaled, by `ParquetTimestampConverter`, which parquet's own round trip +/// `SECOND` column is rescaled, by `ParquetReadTypeAdapter`, which parquet's own round trip /// needs since it has no `SECOND`. Any other pair would be relabelled rather than rescaled - /// `us` under an `ms` type reads a thousand times too large - so it is refused; /// - orc records none: a value is a seconds-plus-nanoseconds pair that `OrcAdapter` builds at diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 60d898511..937e7124a 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -19,7 +19,7 @@ set(PAIMON_PARQUET_FILE_FORMAT predicate_converter.cpp file_reader_wrapper.cpp page_filtered_row_group_reader.cpp - parquet_timestamp_converter.cpp + parquet_read_type_adapter.cpp parquet_file_batch_reader.cpp parquet_file_format_factory.cpp parquet_format_writer.cpp @@ -54,7 +54,7 @@ if(PAIMON_BUILD_TESTS) SOURCES file_reader_wrapper_test.cpp page_filtered_row_group_reader_test.cpp - parquet_timestamp_converter_test.cpp + parquet_read_type_adapter_test.cpp parquet_vector_io_test.cpp parquet_field_id_converter_test.cpp parquet_file_batch_reader_test.cpp diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 89758dc58..76f0df132 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -54,8 +54,8 @@ #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include "paimon/format/parquet/parquet_schema_util.h" -#include "paimon/format/parquet/parquet_timestamp_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" #include "paimon/utils/roaring_bitmap32.h" @@ -75,10 +75,9 @@ class Predicate; namespace paimon::parquet { namespace { -// LIST/MAP do not support pruning fields from their nested value types, but physical and -// logical leaf types may still differ (for example, Parquet reports LTZ timestamps as UTC -// while Paimon exposes them in the local timezone). Compare only the nested projection shape -// here so those representation differences are handled by the normal cast path. +// LIST, MAP, and FIXED_SIZE_LIST do not support partial projection of their nested values. +// Require matching nested structures while allowing the leaf-type representation differences +// explicitly handled below. bool HasSameNestedProjectionShape(const std::shared_ptr& read_type, const std::shared_ptr& file_type) { const bool read_is_nested = ArrowSchemaValidator::IsNestedType(read_type); @@ -87,8 +86,8 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t if (read_is_nested || file_is_nested) { return false; } - // ParquetTimestampConverter explicitly supports timestamp unit and timezone - // conversion after reading. Other atomic type differences remain unsupported here. + // ParquetReadTypeAdapter supports timezone retyping and the millisecond-to-second + // conversion required for Paimon timestamps after reading. if (read_type->id() == arrow::Type::TIMESTAMP && file_type->id() == arrow::Type::TIMESTAMP) { const auto& read_timestamp = static_cast(*read_type); @@ -356,11 +355,11 @@ Result> ParquetFileBatchReader::GetFileSchema() c PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr new_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); + std::shared_ptr normalized_type, + ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(new_schema->fields()))); auto c_schema = std::make_unique<::ArrowSchema>(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*normalized_type, c_schema.get())); return c_schema; } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GetFileSchema") @@ -378,8 +377,8 @@ Status ParquetFileBatchReader::SetReadSchema( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, GetLogicalFileSchema()); - // Recursively match read_schema against file_schema by field names. - // STRUCT supports sub-field projection; LIST/MAP require exact type match. + // Recursively match read_schema against file_schema by field names. STRUCT supports + // sub-field projection; LIST, MAP, and FIXED_SIZE_LIST require matching nested shapes. PAIMON_ASSIGN_OR_RAISE(std::vector column_indices, ComputeNestedColumnIndices(read_schema, file_schema)); @@ -773,20 +772,8 @@ Result ParquetFileBatchReader::NextBatch() { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( - array, read_data_type_, arrow_pool_)); - } - PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - return Status::Invalid(fmt::format( - "unexpected: in parquet, after CastArrayForTimestamp, output type {} not " - "equal with read schema {}", - array->type()->ToString(), read_data_type_->ToString())); - } + PAIMON_ASSIGN_OR_RAISE( + array, ParquetReadTypeAdapter::AdaptArray(array, read_data_type_, arrow_pool_)); PAIMON_RETURN_NOT_OK(GenerateRowMapping(array->length())); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 95fcf2bed..565dec9b2 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -201,8 +201,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { /// Recursively collect leaf column indices for the sub-fields in read_type /// that match file_type by paimon field ID. Unmatched sub-fields in file_type - /// have their leaf indices skipped. Partial projection inside LIST/MAP is - /// not supported and will return Invalid. + /// have their leaf indices skipped. Partial projection inside LIST, MAP, or + /// FIXED_SIZE_LIST is not supported and will return Invalid. static Status CollectLeafIndices(const std::shared_ptr& read_type, const std::shared_ptr& file_type, int32_t* leaf_index, std::vector* indices); @@ -213,7 +213,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { /// Compute leaf column indices by recursively matching read_schema against /// file_schema using paimon field IDs. STRUCT supports sub-field projection - /// (unmatched sub-fields are skipped). LIST/MAP require exact type match. + /// (unmatched sub-fields are skipped). LIST, MAP, and FIXED_SIZE_LIST require + /// matching nested shapes. static Result> ComputeNestedColumnIndices( const std::shared_ptr& read_schema, const std::shared_ptr& file_schema); diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.cpp b/src/paimon/format/parquet/parquet_read_type_adapter.cpp similarity index 69% rename from src/paimon/format/parquet/parquet_timestamp_converter.cpp rename to src/paimon/format/parquet/parquet_read_type_adapter.cpp index 01521762b..6f64462c0 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/format/parquet/parquet_timestamp_converter.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include #include @@ -31,10 +31,9 @@ #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" namespace paimon::parquet { -namespace { -bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, - const std::shared_ptr& target_field) { +bool ParquetReadTypeAdapter::IsCompatibleBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { // ARROW:schema may restore an inline BLOB's physical binary column to its logical // large_binary representation before this compatibility check runs. return src_field->type()->id() == arrow::Type::LARGE_BINARY && @@ -42,9 +41,7 @@ bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, BlobUtils::IsBlobMetadata(target_field->metadata()); } -} // namespace - -Result> ParquetTimestampConverter::AdjustTimezone( +Result> ParquetReadTypeAdapter::NormalizeFileType( const std::shared_ptr& src_data_type) { arrow::Type::type type = src_data_type->id(); switch (type) { @@ -54,7 +51,7 @@ Result> ParquetTimestampConverter::AdjustTimezo new_fields.reserve(src_struct_type->num_fields()); for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_type, - AdjustTimezone(src_struct_type->field(i)->type())); + NormalizeFileType(src_struct_type->field(i)->type())); new_fields.push_back(src_struct_type->field(i)->WithType(sub_type)); } return arrow::struct_(new_fields); @@ -62,9 +59,9 @@ Result> ParquetTimestampConverter::AdjustTimezo case arrow::Type::type::MAP: { auto* src_map_type = checked_cast(src_data_type.get()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_type, - AdjustTimezone(src_map_type->key_type())); + NormalizeFileType(src_map_type->key_type())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item_type, - AdjustTimezone(src_map_type->item_type())); + NormalizeFileType(src_map_type->item_type())); return std::make_shared( src_map_type->key_field()->WithType(key_type), src_map_type->item_field()->WithType(item_type)); @@ -72,7 +69,7 @@ Result> ParquetTimestampConverter::AdjustTimezo case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr value_type, - AdjustTimezone(src_list_type->value_type())); + NormalizeFileType(src_list_type->value_type())); return arrow::list(src_list_type->value_field()->WithType(value_type)); } case arrow::Type::type::TIMESTAMP: { @@ -86,9 +83,22 @@ Result> ParquetTimestampConverter::AdjustTimezo } } -Result ParquetTimestampConverter::NeedCastArrayForTimestamp( +Result ParquetReadTypeAdapter::NeedsArrayConversion( const std::shared_ptr& src_data_type, const std::shared_ptr& target_data_type) { + return NeedsArrayConversionImpl(arrow::field("", src_data_type), + arrow::field("", target_data_type)); +} + +Result ParquetReadTypeAdapter::NeedsArrayConversionImpl( + const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { + if (IsCompatibleBlob(src_field, target_field)) { + return false; + } + + const std::shared_ptr& src_data_type = src_field->type(); + const std::shared_ptr& target_data_type = target_field->type(); arrow::Type::type type = src_data_type->id(); if (type != target_data_type->id()) { return Status::Invalid(fmt::format("src type {} and target type {} mismatch", @@ -101,18 +111,13 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( auto* target_struct_type = checked_cast(target_data_type.get()); if (src_struct_type->num_fields() != target_struct_type->num_fields()) { return Status::Invalid( - fmt::format("src type {} and target type {} number of fields mismatch", + fmt::format("source type {} and target type {} number of fields mismatch", src_data_type->ToString(), target_data_type->ToString())); } for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { - const std::shared_ptr& src_field = src_struct_type->field(i); - const std::shared_ptr& target_field = target_struct_type->field(i); - if (IsAlreadyLogicalBlob(src_field, target_field)) { - continue; - } - PAIMON_ASSIGN_OR_RAISE( - bool need_cast, - NeedCastArrayForTimestamp(src_field->type(), target_field->type())); + PAIMON_ASSIGN_OR_RAISE(bool need_cast, + NeedsArrayConversionImpl(src_struct_type->field(i), + target_struct_type->field(i))); if (need_cast) { return true; } @@ -124,38 +129,54 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( auto* target_map_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( bool need_cast, - NeedCastArrayForTimestamp(src_map_type->key_type(), target_map_type->key_type())); + NeedsArrayConversionImpl(src_map_type->key_field(), target_map_type->key_field())); if (need_cast) { return true; } - PAIMON_ASSIGN_OR_RAISE( - need_cast, - NeedCastArrayForTimestamp(src_map_type->item_type(), target_map_type->item_type())); + PAIMON_ASSIGN_OR_RAISE(need_cast, + NeedsArrayConversionImpl(src_map_type->item_field(), + target_map_type->item_field())); return need_cast; } case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); auto* target_list_type = checked_cast(target_data_type.get()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, - NeedCastArrayForTimestamp(src_list_type->value_type(), - target_list_type->value_type())); - return need_cast; + return NeedsArrayConversionImpl(src_list_type->value_field(), + target_list_type->value_field()); } case arrow::Type::type::TIMESTAMP: { auto* src_ts_type = checked_cast(src_data_type.get()); auto* target_ts_type = checked_cast(target_data_type.get()); - if (src_ts_type->unit() != target_ts_type->unit() || - src_ts_type->timezone() != target_ts_type->timezone()) { - return true; - } - return false; + return src_ts_type->unit() != target_ts_type->unit() || + src_ts_type->timezone() != target_ts_type->timezone(); } default: return false; } } -Result> ParquetTimestampConverter::CastArrayForTimestamp( +Result> ParquetReadTypeAdapter::AdaptArray( + const std::shared_ptr& array, + const std::shared_ptr& target_data_type, + const std::shared_ptr& arrow_pool) { + PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedsArrayConversion(array->type(), target_data_type)); + if (!need_cast) { + return array; + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + AdaptArrayImpl(array, target_data_type, arrow_pool)); + PAIMON_ASSIGN_OR_RAISE(need_cast, NeedsArrayConversion(result->type(), target_data_type)); + if (need_cast) { + return Status::Invalid(fmt::format( + "unexpected: in parquet, after AdaptArray, output type {} does not match target " + "type {}", + result->type()->ToString(), target_data_type->ToString())); + } + return result; +} + +Result> ParquetReadTypeAdapter::AdaptArrayImpl( const std::shared_ptr& array, const std::shared_ptr& target_data_type, const std::shared_ptr& arrow_pool) { @@ -171,7 +192,7 @@ Result> ParquetTimestampConverter::CastArrayForTim const auto& field = struct_array->field(i); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr sub_array, - CastArrayForTimestamp(field, target_data_type->field(i)->type(), arrow_pool)); + AdaptArrayImpl(field, target_data_type->field(i)->type(), arrow_pool)); target_sub_arrays.push_back(sub_array); target_names.push_back(target_data_type->field(i)->name()); } @@ -187,10 +208,10 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* map_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr key_array, - CastArrayForTimestamp(map_array->keys(), map_type->key_type(), arrow_pool)); + AdaptArrayImpl(map_array->keys(), map_type->key_type(), arrow_pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr item_array, - CastArrayForTimestamp(map_array->items(), map_type->item_type(), arrow_pool)); + AdaptArrayImpl(map_array->items(), map_type->item_type(), arrow_pool)); return std::make_shared( arrow::map(key_array->type(), item_array->type()), map_array->length(), map_array->value_offsets(), key_array, item_array, map_array->null_bitmap(), @@ -201,7 +222,7 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* list_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr value_array, - CastArrayForTimestamp(list_array->values(), list_type->value_type(), arrow_pool)); + AdaptArrayImpl(list_array->values(), list_type->value_type(), arrow_pool)); return std::make_shared( arrow::list(value_array->type()), list_array->length(), list_array->value_offsets(), value_array, list_array->null_bitmap(), list_array->null_count(), @@ -213,8 +234,8 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* ts_target_type = checked_cast(target_data_type.get()); if (src_type->unit() == arrow::TimeUnit::type::MILLI && ts_target_type->unit() == arrow::TimeUnit::type::SECOND) { - // parquet writer do not support second, and it cast second to milli. - // Therefore, in paimon file reader, we cast from milli to second. + // Parquet does not support second-precision timestamps, so the writer stores + // them as milliseconds. Convert them back to seconds for the Paimon read type. auto cast_executor = std::make_shared(); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr target_array, @@ -222,10 +243,9 @@ Result> ParquetTimestampConverter::CastArrayForTim return target_array; } if (src_type->timezone() != ts_target_type->timezone()) { - // 1. For nano type, parquet writer will write nano into int96 type, which does - // not contain any stats or zone info. Therefore in paimon file reader, we add - // zone info according to target type 2. For other precision, parquet reader - // will return UTC tz. Therefore, in paimon file reader, we add local zone info. + // INT96 carries no timezone, while other Parquet timestamps may be exposed as + // UTC. Attach the timezone required by the Paimon read type without changing + // the underlying values. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, ts_array->View(target_data_type)); return target_array; @@ -235,4 +255,5 @@ Result> ParquetTimestampConverter::CastArrayForTim return array; } } + } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.h b/src/paimon/format/parquet/parquet_read_type_adapter.h similarity index 59% rename from src/paimon/format/parquet/parquet_timestamp_converter.h rename to src/paimon/format/parquet/parquet_read_type_adapter.h index 2954bfe41..2f2960456 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.h +++ b/src/paimon/format/parquet/parquet_read_type_adapter.h @@ -25,22 +25,34 @@ namespace paimon::parquet { -class ParquetTimestampConverter { +class ParquetReadTypeAdapter { public: - ParquetTimestampConverter() = delete; - ~ParquetTimestampConverter() = delete; + ParquetReadTypeAdapter() = delete; + ~ParquetReadTypeAdapter() = delete; - static Result> AdjustTimezone( + static Result> NormalizeFileType( const std::shared_ptr& src_data_type); - static Result NeedCastArrayForTimestamp( + static Result NeedsArrayConversion( const std::shared_ptr& src_data_type, const std::shared_ptr& target_data_type); - static Result> CastArrayForTimestamp( + static Result> AdaptArray( const std::shared_ptr& array, const std::shared_ptr& target_data_type, const std::shared_ptr& arrow_pool); + + private: + static Result> AdaptArrayImpl( + const std::shared_ptr& array, + const std::shared_ptr& target_data_type, + const std::shared_ptr& arrow_pool); + + static Result NeedsArrayConversionImpl(const std::shared_ptr& src_field, + const std::shared_ptr& target_field); + + static bool IsCompatibleBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp similarity index 76% rename from src/paimon/format/parquet/parquet_timestamp_converter_test.cpp rename to src/paimon/format/parquet/parquet_read_type_adapter_test.cpp index 784eecd05..17933cb82 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/format/parquet/parquet_timestamp_converter.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include @@ -31,7 +31,7 @@ namespace paimon::parquet::test { -TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { +TEST(ParquetReadTypeAdapterTest, TestNeedsArrayConversion) { { // single field need cast arrow::FieldVector fields = { @@ -40,10 +40,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { arrow::FieldVector target_fields = { arrow::field("f0", arrow::timestamp(arrow::TimeUnit::NANO, "UTC")), }; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in list need cast @@ -52,10 +52,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { }; arrow::FieldVector target_fields = { arrow::field("f2", arrow::list(arrow::timestamp(arrow::TimeUnit::SECOND)))}; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in map need cast @@ -66,10 +66,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { arrow::FieldVector target_fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::SECOND), arrow::timestamp(arrow::TimeUnit::NANO, "UTC")))}; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in struct need cast @@ -84,32 +84,32 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { {arrow::field("f0", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f1", arrow::timestamp(arrow::TimeUnit::NANO, "UTC"))})), }; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } } -TEST(ParquetTimestampConverterTest, TestAlreadyLogicalBlobCompatibility) { +TEST(ParquetReadTypeAdapterTest, TestAlreadyLogicalBlobCompatibility) { std::shared_ptr restored_blob = arrow::field("blob", arrow::large_binary()); std::shared_ptr expected_physical_blob = BlobUtils::ToArrowField("blob")->WithType(arrow::binary()); - ASSERT_OK_AND_ASSIGN(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - arrow::struct_({restored_blob}), - arrow::struct_({expected_physical_blob}))); - ASSERT_FALSE(need_cast); + ASSERT_OK_AND_ASSIGN(bool needs_conversion, ParquetReadTypeAdapter::NeedsArrayConversion( + arrow::struct_({restored_blob}), + arrow::struct_({expected_physical_blob}))); + ASSERT_FALSE(needs_conversion); // Only a binary field carrying Paimon BLOB metadata is compatible with a large_binary field // already restored from ARROW:schema. A plain binary field must still be rejected. - ASSERT_NOK_WITH_MSG(ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_({restored_blob}), arrow::struct_({arrow::field("blob", arrow::binary())})), - "src type large_binary and target type binary mismatch"); + "source type large_binary and target type binary mismatch"); } -TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { +TEST(ParquetReadTypeAdapterTest, TestAdaptArray) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI), @@ -141,9 +141,9 @@ TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { .ValueOrDie()); std::shared_ptr pool = GetArrowPool(GetDefaultPool()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, - ParquetTimestampConverter::CastArrayForTimestamp( - array, arrow::struct_(target_fields), pool)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + ParquetReadTypeAdapter::AdaptArray(array, arrow::struct_(target_fields), pool)); auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(target_fields), R"([ @@ -155,7 +155,20 @@ TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); } -TEST(ParquetTimestampConverterTest, TestAdjustTimezone) { +TEST(ParquetReadTypeAdapterTest, TestAdaptArrayValidatesResultType) { + std::shared_ptr src_type = arrow::timestamp(arrow::TimeUnit::MICRO); + std::shared_ptr target_type = arrow::timestamp(arrow::TimeUnit::MILLI); + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(src_type, R"(["1970-01-01 00:00:01"])") + .ValueOrDie(); + std::shared_ptr pool = GetArrowPool(GetDefaultPool()); + + ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::AdaptArray(array, target_type, pool), + "after AdaptArray, output type timestamp[us] does not match target type " + "timestamp[ms]"); +} + +TEST(ParquetReadTypeAdapterTest, TestNormalizeFileType) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI), @@ -179,8 +192,8 @@ TEST(ParquetTimestampConverterTest, TestAdjustTimezone) { arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)), }; - ASSERT_OK_AND_ASSIGN(auto result_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(fields))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_type, + ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(fields))); ASSERT_TRUE(result_type->Equals(arrow::struct_(target_fields))); } } // namespace paimon::parquet::test diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 94d6c99fc..a55951503 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -419,6 +419,29 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) { ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); ASSERT_EQ(inline_descriptor->Offset(), 7); ASSERT_EQ(inline_descriptor->Length(), 11); + + // Python and Java store MAP values in standard separate BLOB files. Validate + // resolved payloads, null values, and an empty map. Rust stores raw values inline in Parquet, + // which is not a compatible Paimon BLOB representation and is asserted separately below. + if (writer_prefix == "rust") { + return; + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr map_blob_result, + ReadTable(writer_prefix + "_map_blob_types", {"id", "f_map_blob"}, blob_value_options)); + std::shared_ptr map_blob_rows = GetOnlyStructChunk(map_blob_result); + ASSERT_TRUE(map_blob_rows); + ASSERT_EQ(map_blob_rows->length(), 3); + AssertFieldEqualsJson(map_blob_rows, "id", arrow::int32(), "[1, 2, 3]"); + AssertFieldEqualsJson(map_blob_rows, "f_map_blob", + arrow::map(arrow::utf8(), arrow::large_binary()), + R"([[ ["left", "blob-map-left"], ["right", null] ], null, []])"); +} + +TEST(PaimonReadCompatInteStandaloneTest, RejectsNonStandardRustMapBlob) { + ASSERT_NOK_WITH_MSG( + ReadTable("rust_map_blob_types", {"f_map_blob"}), + "Parquet does not support partial projection inside list/map: src map UnsupportedReadParams() { const std::vector read_cases = { - {"NestedBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, + {"ArrayBlob", "array_blob_types", "f_array_blob", + "BLOB field must be a top-level field or the direct value of a top-level MAP field"}, {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md b/test/test_data/parquet/append_types_compatibility.db/README.md index 5e4e330be..a1ac373a3 100644 --- a/test/test_data/parquet/append_types_compatibility.db/README.md +++ b/test/test_data/parquet/append_types_compatibility.db/README.md @@ -19,7 +19,9 @@ # Paimon writer compatibility fixtures -The `append_types_compatibility.db` database contains equivalent append-only tables written by Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of types in that table; no individual table is claimed to contain every Paimon type. +The `append_types_compatibility.db` database contains append-only compatibility tables written by +Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of +types in that table; no individual table is claimed to contain every Paimon type. The writers and Parquet implementations are: @@ -29,11 +31,16 @@ The writers and Parquet implementations are: ## Table groups -Each writer prefix has four tables with the same schema and logical values: +Each writer prefix has five tables with the same declared schema and intended logical values: - `_types`: 43 columns and 3 rows covering integral and floating types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE, TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions, VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in every nullable column and row 3 exercises empty values. - `_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, and DOUBLE. -- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ rejects the table schema because ARRAY<BLOB> is unsupported. Schema validation happens before column projection, so the compatibility test explicitly checks this error instead of reading either field. +- `_array_blob_types`: Paimon C++ does not currently support ARRAY<BLOB>. +- `_map_blob_types`: MAP<STRING, BLOB>, including non-null, null, and empty map + values. Python and Java store these values in standard separate BLOB files, which Paimon C++ + can read. Rust stores the raw values inline as Parquet `binary`; this is not the Paimon BLOB + descriptor representation, cannot be read as BLOB by Java, and is intentionally rejected by + Paimon C++. - `_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++ currently rejects `TIME` while parsing the table schema. The tables are separated because Paimon C++ does not allow VECTOR in a data-evolution table, BLOB requires data evolution, and a schema-level incompatibility must not prevent compatible columns from being tested. @@ -42,4 +49,7 @@ The tables are separated because Paimon C++ does not allow VECTOR in a data-evol The `f_blob_descriptor` value intentionally points at a nonexistent external URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor and Parquet `ARROW:schema` path rather than external blob fetching. -Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types differ in these fixtures. The Python fixture records the physical Parquet `binary` column as Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test verifies that Paimon C++ can read both representations and preserve the descriptor value. +Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types +differ for `f_blob_descriptor`. The Python fixture records the physical Parquet `binary` column as +Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test +verifies that Paimon C++ can read both representations and preserve the descriptor value. diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0 similarity index 88% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0 index f12e5f846185605bb3f962505c8337614bd97a6a..f6ca091da9d7caf863f6bd879d0964730206e31c 100644 GIT binary patch delta 217 zcmX>pxI=Km3-)^Mzw)}qN12JhtNvdGkF~4}mpGL)8n&KgtX;cm z=ZuR9o6nRTyfE+T*W-Wpx=x!gGyCxSqp?%iUaLGmoTQ}r&FTCh55)p0t~)?Rr`M!I zJL~TM6Fum3UXPSnIv7J+(F)$25i=hXHTO5N;U?|EjK;oK0vr>Uxf44f*A52G%qY?FMy=~km>aZy9W dj5MJKT<4e%98f4uFffpiSaYC3Mu7?4M*!%+Z#@72 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0 similarity index 95% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0 index 034d68e1753aa64420802cffe1af82ab1fd4adc4..4b7ac1f09b5ab3a881163116777255588355951c 100644 GIT binary patch delta 24 gcmaFI{*HaaJ!S#ZiO!|Ub$j04n0)?1-HpW?0g>Ac2mk;8 delta 24 gcmaFI{*HaaJ!XNG0^hX#oOT_17qZ3E^Em%<0FMd_djJ3c diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 new file mode 100644 index 0000000000000000000000000000000000000000..b7ae3ef2cea9175039f7a0e874fad41fbfd4d7b4 GIT binary patch literal 1115 zcmbVLO>fgM7|yDk5t=5%6`wdXB~{zOddnK4MId zY%aW_l!Kza0z5WMh-}S*D@e{+l#Vx1j`5;`inBG0?O90Eb*Q6K(^=_ViPWJ9-5X$^ zY6I7BMFixyn!8f{M|lWs60fMoGHf)|Os{%ef68h8S9(i(egl;4nI`J>J-c$sW00Q5 zF%W$(>c@=Xsi-F6G$|*w*a!2DM#ZQO2o3Rh%t5zg7Nep%ET?w_Ee#p^HRYC(4!zzJ z=q^D^mvZ$?W!m4Jt~a_C_y5SFVsHM}1sS2i z?q$w#N-!hR=j6lJmtWqPXSeTt@^$}J?cL7(_YZzbM{jG=Ygs-?a2ka)=SrLS?G9E0 xB|K=gl(yP#DUVb_m9R-VxYJZ09_;Tc>JK1Gcg3nn;#w}bCVl+YlJ?fkegUNeVOIbE literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 new file mode 100644 index 000000000..0bce94c9f --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 @@ -0,0 +1,28 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_array_blob", + "type" : { + "type" : "ARRAY", + "element" : "BLOB" + } + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "blob-field" : "f_array_blob", + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788859707631 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..29a83ff0b --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "829d0558-0c81-4701-ba81-108a84c1de2e", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1", + "deltaManifestListSize" : 1115, + "commitUser" : "389ad12e-4421-4e4f-ad11-14284923cc57", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788859708818, + "totalRecordCount" : 6, + "deltaRecordCount" : 6, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2d2cb3152d59b7a0c18bdcb044dcba1b6e638815 GIT binary patch literal 298 zcmX9)!AiqG5S>N|dr(2>ELqq?E)5Yh?k1A7(2Ex-o{Hc#nrt>0Fl|k(U*d;&@dNw; zKchb(f)jIi%)H0$do#1VzS9Wkp~A0^{m+R+mncI95Pn|%!r~EN1ON`E@=jV&#P0_UL)x*|2*|tE3)CqV_Q{4k_S4bfPko0l_O!P7yahKol zJYe&|b%@{){$~t~LTEa&f;qXok8sRujDY^R0pt0YhQ8et-OJ9tUbLP6X1i^(UimuG zX{7wH+%?aYf2CrROtmse7FY2sRf*M8on^W(wydo&nJ#Ue>N6oFZSzf7(v|)Ib$BuC literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 new file mode 100644 index 0000000000000000000000000000000000000000..c803b1c9b838d44b20028bba77ca94aacb7cce19 GIT binary patch literal 2233 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~8U1%iy8Q%J8W}UZY{Fu1vLvyIaY0p<@%YGL!?v zf8Bq3bZJ6D)&Kn6`Q=MfWgqT~nxeugudHDuG{xfcg?su=QyqI2>`;->SN5v@U#G~- z!o9&XVpltRk})Z9mS=;u1wR9G^O?qj2NDuCFfkW3aG-kt0Nu9S Aga7~l literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0 similarity index 86% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0 index 071447210821caf9daa6f0b6758a08ce1b97e5e7..655f831bfdce95b75c23df5f03ffb3ec87cca831 100644 GIT binary patch delta 24 gcmcc4@s54NJ!XMzxyMRMZ2wN-;5z(F;Hps{0FY)2bpQYW delta 129 zcmaFIew|~(J?2WChr0a@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z1_ASanV*xNaXctB<(!_e_cGHKMvblde;GV_nHXAFSmJUM^D@&?i%WFPjFL@EQJtb2AGQLnB>-Pf9Ed5AEo5dzgf=G%;&z)&HB&%CMJ}L8wbvV}j{f#@D-d z?=615Z=1b^>+Ef9$@bpf-jCjyRGs{GDer`!Vc0L}Q*)W-_Zv&MwkX`Z({lcdzyE*s zmxl7`TB|g->TKFB6T4+$w9nzyG54jS?T^38^%J}_F`W6(^ZDG}scsF-4gZbjTPh^D zm^C&$aSYrWv2;=G)vdp!j5Kb$y_hw#G;^m~=;UqFCay8JwG`rG@D+BrRMGO8`Gh!I g@U1sf<_N}w3%MQS>HEqad0+~IaE$|>03*7m0MK=3CIA2c delta 275 zcmew%I9F(c2uHoy=1qmhz1|E9O?_9~p5Yh5c7|DFtN!1FAjYR`42yWQG#WI|GJbBf zVSC1`cg!(iqheSBbNP2MPYo~j?AzYnrZIkH+CxF6rQs zgIhUDr7hh5aZRZC=ea=OP^$xFE zyLxA9u+J&}x{o%NLc$EvDvky5SqyU9He@oX7bmbC V|JAIM;MKrsr@-C;_Bl6_*8w{}a5{zXenYM#mMzB&0d0BI!(WdHyG delta 24 gcmbQwHlJ++KeND`(2iZJy99Jv-mUn`r@tZq0CP|Z1poj5 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1 similarity index 85% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1 index c85847cb0f08b8caaa7342ef372ae0841c60c9b2..cf8d9f9c5614155b3c30959b099f19bd8a9eada8 100644 GIT binary patch delta 104 zcmZo>Z)V@X&n(cTy(`__aN+$K`$T_y|8^>VvOKe-w?(Q&Vp3Y7fo`ICTB@#znVBV! sm6WQRoMK>Unv$AonwFTXYw$stg+YRWi2;b1SeO_L_$`b%7 delta 104 zcmZo>Z)V@X&nzHQR>*A>EVJfuwa%~I=S?q6mS>jqHb_k~OSLpL(lt*tHPbaQG&a?> tOiVJ+O)*O~x3n-aNJ&de(>2(o!p0!Mz{CJVOe{.", + "timeMillis": 1788859561835 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..ac1f6f1a0 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0", + "deltaManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "657b7800-ea12-4a26-8075-4b2fd4e20d51", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788859561849, + "nextRowId": 3, + "uuid": "43460238-f728-4ee1-8600-1f3d0c13234a" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d733aa4d5f993b4af7d2a11de590694bd6e38e19 GIT binary patch literal 457 zcmaJ;OH0E*5T2xA>7j=pPS}+_M9@&A4~z7m;9*h}#a9~8vYX6D9!mAhm zo8Zh=!IJ~?`ex>vS$0TS4+GS=b&3jM1)%xS{fZkofCm6RYmF5F>$uk9mXK9h9SEue z>4~8ECu6`d1nWFq+W-tqs}EYu$M=(m9DP846SP0QgGd=`YyuX2(+3qAL9r33=*^2h zz2rQp|F_4q_-${ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob rename to test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 new file mode 100644 index 0000000000000000000000000000000000000000..f238c7da6b64c69b7f59d2c45d93fca974df54db GIT binary patch literal 2298 zcmeZI%3@>@Nh~YM*GtY%Nlg~2DlSRPOGzwB5d#YrCugMQCSH@TRw}71NL8{@P%27I z&M!()(os;#OUwm}_$KCMrll5_xaO4ih+uwtKo{`!(4-cJ^lSa@-U+^ z^GcLzbrg^kd3*Y~;8zjg7!=|ef@V-sWl3r=no&-n&fcye_zes34{`K~ho}t3uhh-c z#}!EhzVL8KEJ<|B%t`f4ElC7v!5SPO^Wyy+eUS}SDlRF?%u7e}2v}vXXB3LcoO}#* zLH^;9%FlP**=^ zU``12b#e_tPLtSDE-*=wrX$$d!`0U@-V>!z!D6zHYnZDKvhA2flWRmskYhY3Ga{!Q zL~LRt8F+3+O%={Tu8yGcD?Y>%CG%rRg46SR!N(F#cVG{D9R(fMbXURzE{a zZ$bvSc?JcCfa3xsS;8GlTttKic>-gU${q%mV6n9fS4wog`AzV(ux&fu^m?^LJWCt1 z##a5m3C#>=Ss9#8DQhfHf9dpp|G#|JyX+q(pNlFw61lOwe&WJYb9}$ui~d&oqH4+W z+>;ZmS>L<9Y)NcUVM_6P^8d`ApFi~r6siJ$IqcF|_qdTg+UFMc#&wkB8GJNxvO=Z8TavS9D75Ht%Z4-cBdBDiEJecqh|QBD~+ z>H*GzoQI&_0~t-56HK!>VucvS3BycKS&L$pghkDd=PQnkknQM}DeKF$I1`}i_gc_U zHxMx)Ms2JRJkBnZ+a~=n;ahf;Lmz)JlR~o$t5*_nqH&D$UCTwFa-<-j2~Tk+FK?Vd$99HP=Unp3y;F z_?Q~fcyNX2gEZ|Tqvx7*CQUoAJ35h}W3`Zn%a)B~gke}~S;#iMfUggy9H=i&XSouXcz>znr(K;9;{?{+`ZHGw2vlhj3b{M4EK zViV?y>>iWPL&)S|tH2ZsS;Q2xdDcJ$#Im_2lYQHwhz)`yf{&y~BD=ySj}o&zJ^`HC B4qpHO literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 similarity index 61% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 rename to test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 index e311af948..08ed496fd 100644 --- a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 @@ -9,15 +9,6 @@ }, { "id": 1, - "name": "f_array_blob", - "type": { - "type": "ARRAY", - "element": "BLOB", - "nullable": true - } - }, - { - "id": 2, "name": "f_map_blob", "type": { "type": "MAP", @@ -27,7 +18,7 @@ } } ], - "highestFieldId": 2, + "highestFieldId": 1, "partitionKeys": [], "primaryKeys": [], "options": { @@ -36,8 +27,8 @@ "manifest.format": "avro", "row-tracking.enabled": "true", "data-evolution.enabled": "true", - "blob-field": "f_array_blob,f_map_blob" + "blob-field": "f_map_blob" }, - "comment": "Generated by pypaimon 2.0.0 with ARRAY and MAP.", - "timeMillis": 1788513128455 + "comment": "Generated by pypaimon 2.0.0 with MAP.", + "timeMillis": 1788859561850 } \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..abae40fe9 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0", + "deltaManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "cadd033b-11dd-40e2-a759-0306ab9df091", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788859561854, + "nextRowId": 3, + "uuid": "0279865a-f61e-41bf-8218-69adf804978f" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 deleted file mode 100644 index 8e955da73..000000000 --- a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 +++ /dev/null @@ -1,15 +0,0 @@ -{ - "version": 3, - "id": 1, - "schemaId": 0, - "baseManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0", - "deltaManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1", - "totalRecordCount": 9, - "deltaRecordCount": 9, - "commitUser": "297eeaeb-5c62-4b3c-8f9e-950b09612bc3", - "commitIdentifier": 9223372036854775807, - "commitKind": "APPEND", - "timeMillis": 1788513128459, - "nextRowId": 3, - "uuid": "d2837b32-27c3-453e-8470-ccb9d176d3f0" -} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f31d5eccfc97ca52a0f65fd798a39ab165935c92 GIT binary patch literal 1124 zcmaJ>UuzRV5TDx|XDIj3mhN&V_YlDzMbb8nrWDOX_nI{QXYGY1X@VFMO=98F)TB1; zoBAYv1O*>`6cKz8{5r)Cq2SDJThk&A*`1w#zxmCCWJeYkxXB%UXNy}LCIM!?&Hql= z4FCfG7@im&4m{3pCNX6kk9D49-olL7Z9sr!zPyF%5+Lyd;H{07&8XL$ZS?z%)7ezo zfKMMb5i7ydNj`Q3Yy9GM{%!A*2dgTwZpl>jo#vD?Mjv4I_jyMBuPc!F6D{6IH2Sj4DJ zs0p++F#&Fp)9D3-o|NlFKSo2C&yJqPopa6^3|D6x)po zwhz{6ezk}EQ11up^W;ZK(S>nIOFWkqy^0R;Lg;}G{f+BM6;yCoqarmW4Mk4;o>ZM0 z;5uKG^k2RR0*-sw=)ZW`I-2bd634CnV83^e$Slp>n@hP#4*)+f#GmXA{=fVI6-M+1 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 new file mode 100644 index 0000000000000000000000000000000000000000..0d557ef9d11550fdfb760dd7017db945ce56d6d4 GIT binary patch literal 2421 zcmds3PiWIn9PT#%943m}{Da^aPmmqD>S3bn;$_26*3cv4{^=uYmacoFrYAgCZ-1W&WRByEy4Ypbwbyp{L9e81oC%lG@5 z$oToX9s9*8O$}t@6F4o-gAt%i$};fJ06mw%pbKa?u4)P32a1@+*^C%B4pl7`5XtWe z9LR`LT2%rjNeU}i6LnmIyB|+VSWaX?*T{^?M6OT_a#1?M67FIxNzp-`lr~BkfPR_{ zRlXgdIG*O$5sc?_oLxuJBLw>)UzvoD@DzhARJ1azmu8@SySdsRR3zU(A>#(IRM@Ev zvbrWIsdY1&s3<+`Ldj|whKmfMU_@YDQg|?7nnnHatUV|2${D7HYC#ACit-fisJS(s zvUEd6J9fPdBs7MKQH%hcek#mW8^sC?W2xwnd4>};P0SGuB^{>?zZVw@m~xNiNBtmy zlj5YTgDzv$@*5nvR={k^p3mBo66IF-5Lsf`75_g)4MRlX83D3EvRgv92Xc<%y*oh+ z|M%!jxr;`@tFOh%e-ZWfjcyV9Hfj?L_F=v4UcEc zzWl_MDJd?>ycPDkL^56%)0vL6B+C+sQjc9--V8wo@(_8WrBtlw`Etj*C?JtP6oMgh zU_{~`4%yP!ELIFv>%>aHh`Jh?Z%_N_Jck59xgrMPQ0X+GU1=yJNIgbdmEk|ZCu)w+)IzWxA>cOaPn literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 new file mode 100644 index 0000000000000000000000000000000000000000..760835cecefc946e4d892abead3b2bd1d4017da0 GIT binary patch literal 1117 zcmcIkK}*9h7@Z29MZ_PF^Qc$Zty?1&+fLSY4#x;tn{AagEonBGV+RjjJPZ0GJbLi# z-5=sV@Mu!!TDKu{;3b6QeedPH?<4i@)z9n*$%us8rNitx z^#&xucYAF>hhYfH6Trs_0TsZA7c(D}0mTtX6YR;unDrGb_5-3MIEZ4ZWF&n;3B#Ox z!;9s;fP{VuD(y+eB(6{sHf+6a3GSIzKslHSX(=TEC%OTVrJ68@q?`q`zl^f=%K|El z7ciDGkOS_T>{|d~DeO literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1 similarity index 89% rename from test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 rename to test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1 index a01a7456a4d0d6449ba40f5f9e668ff09d458aca..c025a6df87d5c5e71eda9e586442edfb859c3d13 100644 GIT binary patch delta 126 zcmcc1ahP+1FH4zy^5M^Nv(CwLZqRw(7Qbm8(=tYlt@?i%B0`xMo-i}9#N{UDWu~PT zm*^Usr&uN%ni%RPn;Dtuni!iT>n5cn8|a!Em?bBhr5UFfn560&yi#RiV3J^9Vqjom RVP@Fbn;m^-`;y=m0GBWf*Z=?k diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 new file mode 100644 index 000000000..c090dea09 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","row-tracking.enabled":"true","bucket":"-1","blob-field":"f_array_blob"},"comment":null,"timeMillis":1788860192967} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..51c4b5d99 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-0","deltaManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-1","commitUser":"a8381718-0f79-46d1-92ec-2879a4cca6f8","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788860192978,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..08aadfe58f748d2fc80f60e9858d51eaeb840a6d GIT binary patch literal 1503 zcmah}&2JJx6rUXyB!)&xI+NXom}=~X7}`=&2sK=0TcGqy*c7lm*esM%6bb?@tp`&* znCPF;c=qPqqest;e}#7&-h2?_92=lV(FogJFc?MDH1Bh@vA^`A^Ep5Uv0+{?jhzf~~>S50fXQy5}>H*3O zi-rN}CW`;PBThg-xzQ@mHSP8sLIdJ`7Wo_T-uW2f255SK>Y9oz z!67@MItN9X5lPqF4o9%7y5XUEoKh766kUTH!ocp4_A~_{dXR6kBtYQpN?|*Q#sjEb zQ)WqR8nYT}KD*Ot_4m7#qiWM;Di~*pM(YO!LSj_Ad$|n=to-{}hYQ>1huOl?`aSwma-zmnn;SD++ zJMD84lKnf{5i8~foP{HMyM_;(kZCy45EP!l(J4w)e%sO04r3jd@SSBmjb z3C7isGf7Rwx!zJC$pe1|O!#{@gPFOQ`M+{b!?NHTeyJGO)c8z;aZhMs>_4H6tG5!E zzVVWTUm}CADvJ{SgL04I<=BZ!beCrGL$c7>V8b4LgS5nG4{x)9r_f*)iRHxw(=m#1VQ zi!LFxb`G{P>4JPmR(E9(P!Xbtn~Cu&6^u^_Q9yF5HIW;%lpXLdVCbk`K0m?}TQ49p zkw*VPGu=HX$ENDAKz;>8s&_2XqTX- znZ@}_@~OV_D3G5*J4ggK@WZHZld`fVj~dPepu3Lik6+Sp-q&pJVXSOTz6XH9|3v2D z{73cRDp|^)fAWP$qUmkB^XjD9o9lFEPOF`6y>&bjOUy6LNAx6o8k`?FgKuvRKJC8& D+{ix< literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 new file mode 100644 index 0000000000000000000000000000000000000000..72f09905281682bdcf2d3c9a6d4e2c0bd15b795c GIT binary patch literal 2418 zcmds3&ui0Q815?04b+KFk?kViX-ZpRC*48F+HWj0ZD*Qn)iFQP_FEUpkCJASZFG|% z2>xaV-9a3iA|m2NckwiFhan6E{{Rv7(39Z7okYJRZIU%>tFT?Xl#=hs^S&j=Hm8p^0yoFSibUc-fgs47Up6Ik;5j+w`W%ovu18(bS6lqL&XxAy=D(OK=o}EL5^K ztdC}(eY&&SAXFybMj_(_#i^K68x(X+RB~HpG*Ln2iq)A1Qyfc~k>GgD zzUV4%*hfj; zGLVDF8!eS$WzSbT-bDe4{E-NZm;)mc_b_EkW3yN>RId{&0TVnMb~4a@-B7sR%dQn8 z%gs2lR~Q<~Uyq&KCuHN~H)eq1`n|K2U7M`(3ibfu`JO5Y?PZ2*VSwYMCPTl!m8m)h9l zTW&rM?wu_CUPRw#`SB~IOPi+x3Iz9<;2VUE_lMn_KfM-$tJ{wfasR ZdAKjN|LV=R!s^(UX`z<>fLqsYT>1^- z7x)YQfm`>^#M;ERlv;2z40p~wXU@6TYTs^b9peE_WH0fF$ImFGB+G&%Mj;uHaI>;s zJn!9Q497mE{y|>MydH`0{caP){V)X52(V#FKowBpCDaEJh;c-+6nmnW(5{U6y+`B} z2T>BsIZ2nqgkna1;N|vCKtewQ)#fN?;!mgn>$YZ@oO_}b#0-qNw2%^jGfjucR18=^ zGDd^AJCCxpn-VHaW-z8}AVpQ7nseF(-$JAcb?B^sy;SOs=5Pzhb`+;z{htsKn%7&9 zs-c;vq3A9w{sY=WMzUX_8OrqvDAUz-RI^-jnK?`;9`Zy3Mj|ob$p(Qu+Sa%w0iSem zKV+aPc5A$#RGPV6JBNmXG$Y;$ nx^-2KWa{=ipIx wlZ*@ubj?x{%~A~w3{#U*Omq#Ns4_7yNiZ-mFfg$&Gwke5kek5Pz=Uof0ATwgTmS$7 delta 105 zcmX@id6;v9FN?tYvW2R@r?i*waGZV1a;d{?ax9CKk3ouIVzObfv97U!L7J|KXXaCVMH+(jOGq0+k1wq0)jK+6ru0 z_Q9;i2i<=`W6Z|*xbejo6W{#96l7^h z=)w~3?g&G$5My&cExwEd9mX`q7>CCPk62E=V?;QpJ_r?7k*scH~ zzcBV?5HjoTd41mLbezF_JfX24zTO6`h)Bf5-pZ-G#>}Rx5e@p?(|oT z=@8&Hj(dLCJFSyuZ-^MU8Icp>78^n2*Gv)d){to1xtK?2KzvigW6<&$B{kJ=aBvd^1@6o6R7of8Iar9yJ?n2kuVrS!VIpM`gy6 z+zK8a((N4LMF}p9BoD7ak)#eZTiu@EOay1SHPLVz?Z#OTb0&`uL8xH{PwL^A%9wzF zPOH%sIwmp8ciI@tvYjw^3B-TtxDOi)6baiqQ5FV%NoGvhNJj)s!~D?vT#M$Tfv^%& zDQmJhl2%#M)69Gvg!I%FI6v0n+*%nTJ>O zIW-MxHg2pJx*if{elY>%{_u;`|CLjcQ^8NoXIk{R9{t^i?cSnxhXZeon>P}eAp$tZ zDEW8D;F$ppoB#MsBN{OO(r@E%T0vPV?N#4X184neEaFP^*QJz2cyMh?*_Z1latn`O zmoi1?V1g}W4lOtu?qKc0lg=SRT4vyzp!N{@5-gcQ5DFqIQ|r@?FDxwc={t*ycj6eo zi8j>tR{n@Suv7|xReWI|;#W^FK1=>#d?u47n_flqr;y5Kty$l!xOfU6Wv`!t$1~^I zW%OU?m$50eG9LrdD!H6QB`GDcO1bs3(!i-!9!q(ScKaZ))I7@GZPhpKw(tOx>9zY9 z`I24=pIRB9pT4rGRN2i|N|oYDt?jz?jnc7G1>&ik+#tR=ndvLNAwPrX;H0L(59_M` z-F>nN^#lA|DggCIQi4tZxGd9sRT6ZkOL{FoLz#quaYauWuiuSIuU1XQRXyxCJ2+}* zlLv{dZaqQmd8i1zo*d9Pi&s8t`G9wgjT@IJtKp?gj3 zs{4j~NO^!rd3>pPqM<@lHVGdHP~47`E${t6z#u+IPh diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 deleted file mode 100644 index 286b07f38b313aa7d9884d28750cdc6baa602d40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2432 zcmds2PiWIn7_VixgC4YuIRxkSE(&&45ca3AL)N}+acMh~w5yvVOWU_Dnl@dMP39B^sspl1aXQd(U+u6vSw`+=EX}X@B7~G`@Y}% zeZNE!C$?;D=O<;!my}W{iGMRW87Yb&WjPT|BGK)tv%;R!ippnGyqv1n50&H?%J7%K z5Xer5BJfTFbt;E|AIK;v$tmCkSw4f59G}zhVg=2YD&7VGG5x_H4C(`O*zRH6q{c)sY^YKv zh66^TbindJ^SZ`xrOUPz!^=%sv=_IWd+s^+{-uLI zUGdHum)x_v`SL=?ueG)Dj^~SwFJHYCm#?X(21AJhR~o2=J*`bGOLfcAgPNzEGc)6R zZ_Ndcv;`Y?Tv%B?bNI*WU8|2iPrm=OeZWSMPq0F{q}!w`U2hUH^D;qKR1_H{kF|@)6<~VzPb3$J=V7G{D Date: Fri, 11 Sep 2026 10:26:20 +0800 Subject: [PATCH 2/2] fix(parquet): align read type mismatch error --- src/paimon/format/parquet/parquet_read_type_adapter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/format/parquet/parquet_read_type_adapter.cpp b/src/paimon/format/parquet/parquet_read_type_adapter.cpp index 6f64462c0..8ba47dc0a 100644 --- a/src/paimon/format/parquet/parquet_read_type_adapter.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter.cpp @@ -101,7 +101,7 @@ Result ParquetReadTypeAdapter::NeedsArrayConversionImpl( const std::shared_ptr& target_data_type = target_field->type(); arrow::Type::type type = src_data_type->id(); if (type != target_data_type->id()) { - return Status::Invalid(fmt::format("src type {} and target type {} mismatch", + return Status::Invalid(fmt::format("source type {} and target type {} mismatch", src_data_type->ToString(), target_data_type->ToString())); }