diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 009979365..c008d222d 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -173,7 +173,9 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( case arrow::Type::type::LARGE_BINARY: { if (BlobUtils::IsBlobMetadata(key_value_metadata)) { if (!allow_blob) { - return Status::Invalid("Blob field must be a top-level field."); + return Status::Invalid( + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } break; } @@ -258,7 +260,9 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::LARGE_BINARY: { if (BlobUtils::IsBlobField(field)) { if (!allow_blob) { - return Status::Invalid("Blob field must be a top-level field."); + return Status::Invalid( + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } break; } diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index d644e5635..283b61ec2 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -170,7 +170,7 @@ TEST(ArrowSchemaValidatorTest, TestInvalidDataType) { } } -TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { +TEST(ArrowSchemaValidatorTest, TestBlobFieldPlacement) { { auto arrow_schema = arrow::schema(arrow::FieldVector({BlobUtils::ToArrowField("blob", true)})); @@ -186,14 +186,16 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::field("nested", arrow::struct_({BlobUtils::ToArrowField("blob", true)})); auto arrow_schema = arrow::schema(arrow::FieldVector({nested_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto array_blob_field = arrow::field("array_blob", arrow::list(BlobUtils::ToArrowField("item", true))); auto arrow_schema = arrow::schema(arrow::FieldVector({array_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto map_blob_field = arrow::field( @@ -211,7 +213,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::map(arrow::utf8(), arrow::struct_({BlobUtils::ToArrowField("blob", true)}))); auto arrow_schema = arrow::schema(arrow::FieldVector({map_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { auto nested_map_blob_field = arrow::field( @@ -220,7 +223,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", true)))); auto arrow_schema = arrow::schema(arrow::FieldVector({nested_map_blob_field})); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } { std::vector nested_fields = { @@ -230,7 +234,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { arrow::field("nested", DataField::ConvertDataFieldsToArrowStructType(nested_fields)))}; auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields); ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 19fbdb136..0dd2533d3 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -789,9 +789,9 @@ Status SchemaValidation::ValidateMosaicDataFields(const TableSchema& schema, const std::vector inline_blob_fields = options.GetBlobInlineFields(); const std::set inline_blob_field_set(inline_blob_fields.begin(), inline_blob_fields.end()); - // Match Java SchemaValidation by validating only fields stored in the normal data file. C++ - // permits BLOB only as a top-level field; descriptor and view fields are inline, so Mosaic - // must reject them here. + // Match Java SchemaValidation by validating only fields stored in the normal data file. + // Top-level BLOB fields stored in separate files are skipped; descriptor and view fields are + // inline, so Mosaic must reject them here. for (const DataField& field : schema.Fields()) { if (BlobUtils::IsBlobField(field.ArrowField()) && inline_blob_field_set.count(field.Name()) == 0) { diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index fed7ea5a5..117555782 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -1245,7 +1245,8 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { }); ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, nested_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options), - "Blob field must be a top-level field."); + "BLOB field must be a top-level field or the direct value of a " + "top-level MAP field."); } TEST(SchemaValidationTest, TestMapSharedShreddingCompression) { diff --git a/src/paimon/core/table/format/format_table_read.cpp b/src/paimon/core/table/format/format_table_read.cpp index 7bc14b59f..99dda9787 100644 --- a/src/paimon/core/table/format/format_table_read.cpp +++ b/src/paimon/core/table/format/format_table_read.cpp @@ -60,7 +60,7 @@ namespace { /// `table_unit`. A file's unit means something else in each format, so the rule does too: /// /// - parquet records the unit, so the file's is the one its values are in. Only `MILLI` under a -/// `SECOND` column is rescaled, by `ParquetTimestampConverter`, which parquet's own round trip +/// `SECOND` column is rescaled, by `ParquetReadTypeAdapter`, which parquet's own round trip /// needs since it has no `SECOND`. Any other pair would be relabelled rather than rescaled - /// `us` under an `ms` type reads a thousand times too large - so it is refused; /// - orc records none: a value is a seconds-plus-nanoseconds pair that `OrcAdapter` builds at diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 60d898511..937e7124a 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -19,7 +19,7 @@ set(PAIMON_PARQUET_FILE_FORMAT predicate_converter.cpp file_reader_wrapper.cpp page_filtered_row_group_reader.cpp - parquet_timestamp_converter.cpp + parquet_read_type_adapter.cpp parquet_file_batch_reader.cpp parquet_file_format_factory.cpp parquet_format_writer.cpp @@ -54,7 +54,7 @@ if(PAIMON_BUILD_TESTS) SOURCES file_reader_wrapper_test.cpp page_filtered_row_group_reader_test.cpp - parquet_timestamp_converter_test.cpp + parquet_read_type_adapter_test.cpp parquet_vector_io_test.cpp parquet_field_id_converter_test.cpp parquet_file_batch_reader_test.cpp diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 89758dc58..76f0df132 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -54,8 +54,8 @@ #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include "paimon/format/parquet/parquet_schema_util.h" -#include "paimon/format/parquet/parquet_timestamp_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" #include "paimon/utils/roaring_bitmap32.h" @@ -75,10 +75,9 @@ class Predicate; namespace paimon::parquet { namespace { -// LIST/MAP do not support pruning fields from their nested value types, but physical and -// logical leaf types may still differ (for example, Parquet reports LTZ timestamps as UTC -// while Paimon exposes them in the local timezone). Compare only the nested projection shape -// here so those representation differences are handled by the normal cast path. +// LIST, MAP, and FIXED_SIZE_LIST do not support partial projection of their nested values. +// Require matching nested structures while allowing the leaf-type representation differences +// explicitly handled below. bool HasSameNestedProjectionShape(const std::shared_ptr& read_type, const std::shared_ptr& file_type) { const bool read_is_nested = ArrowSchemaValidator::IsNestedType(read_type); @@ -87,8 +86,8 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t if (read_is_nested || file_is_nested) { return false; } - // ParquetTimestampConverter explicitly supports timestamp unit and timezone - // conversion after reading. Other atomic type differences remain unsupported here. + // ParquetReadTypeAdapter supports timezone retyping and the millisecond-to-second + // conversion required for Paimon timestamps after reading. if (read_type->id() == arrow::Type::TIMESTAMP && file_type->id() == arrow::Type::TIMESTAMP) { const auto& read_timestamp = static_cast(*read_type); @@ -356,11 +355,11 @@ Result> ParquetFileBatchReader::GetFileSchema() c PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr new_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); + std::shared_ptr normalized_type, + ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(new_schema->fields()))); auto c_schema = std::make_unique<::ArrowSchema>(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*normalized_type, c_schema.get())); return c_schema; } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GetFileSchema") @@ -378,8 +377,8 @@ Status ParquetFileBatchReader::SetReadSchema( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, GetLogicalFileSchema()); - // Recursively match read_schema against file_schema by field names. - // STRUCT supports sub-field projection; LIST/MAP require exact type match. + // Recursively match read_schema against file_schema by field names. STRUCT supports + // sub-field projection; LIST, MAP, and FIXED_SIZE_LIST require matching nested shapes. PAIMON_ASSIGN_OR_RAISE(std::vector column_indices, ComputeNestedColumnIndices(read_schema, file_schema)); @@ -773,20 +772,8 @@ Result ParquetFileBatchReader::NextBatch() { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( - array, read_data_type_, arrow_pool_)); - } - PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - return Status::Invalid(fmt::format( - "unexpected: in parquet, after CastArrayForTimestamp, output type {} not " - "equal with read schema {}", - array->type()->ToString(), read_data_type_->ToString())); - } + PAIMON_ASSIGN_OR_RAISE( + array, ParquetReadTypeAdapter::AdaptArray(array, read_data_type_, arrow_pool_)); PAIMON_RETURN_NOT_OK(GenerateRowMapping(array->length())); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 95fcf2bed..565dec9b2 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -201,8 +201,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { /// Recursively collect leaf column indices for the sub-fields in read_type /// that match file_type by paimon field ID. Unmatched sub-fields in file_type - /// have their leaf indices skipped. Partial projection inside LIST/MAP is - /// not supported and will return Invalid. + /// have their leaf indices skipped. Partial projection inside LIST, MAP, or + /// FIXED_SIZE_LIST is not supported and will return Invalid. static Status CollectLeafIndices(const std::shared_ptr& read_type, const std::shared_ptr& file_type, int32_t* leaf_index, std::vector* indices); @@ -213,7 +213,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { /// Compute leaf column indices by recursively matching read_schema against /// file_schema using paimon field IDs. STRUCT supports sub-field projection - /// (unmatched sub-fields are skipped). LIST/MAP require exact type match. + /// (unmatched sub-fields are skipped). LIST, MAP, and FIXED_SIZE_LIST require + /// matching nested shapes. static Result> ComputeNestedColumnIndices( const std::shared_ptr& read_schema, const std::shared_ptr& file_schema); diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.cpp b/src/paimon/format/parquet/parquet_read_type_adapter.cpp similarity index 68% rename from src/paimon/format/parquet/parquet_timestamp_converter.cpp rename to src/paimon/format/parquet/parquet_read_type_adapter.cpp index 01521762b..8ba47dc0a 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/format/parquet/parquet_timestamp_converter.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include #include @@ -31,10 +31,9 @@ #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" namespace paimon::parquet { -namespace { -bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, - const std::shared_ptr& target_field) { +bool ParquetReadTypeAdapter::IsCompatibleBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { // ARROW:schema may restore an inline BLOB's physical binary column to its logical // large_binary representation before this compatibility check runs. return src_field->type()->id() == arrow::Type::LARGE_BINARY && @@ -42,9 +41,7 @@ bool IsAlreadyLogicalBlob(const std::shared_ptr& src_field, BlobUtils::IsBlobMetadata(target_field->metadata()); } -} // namespace - -Result> ParquetTimestampConverter::AdjustTimezone( +Result> ParquetReadTypeAdapter::NormalizeFileType( const std::shared_ptr& src_data_type) { arrow::Type::type type = src_data_type->id(); switch (type) { @@ -54,7 +51,7 @@ Result> ParquetTimestampConverter::AdjustTimezo new_fields.reserve(src_struct_type->num_fields()); for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_type, - AdjustTimezone(src_struct_type->field(i)->type())); + NormalizeFileType(src_struct_type->field(i)->type())); new_fields.push_back(src_struct_type->field(i)->WithType(sub_type)); } return arrow::struct_(new_fields); @@ -62,9 +59,9 @@ Result> ParquetTimestampConverter::AdjustTimezo case arrow::Type::type::MAP: { auto* src_map_type = checked_cast(src_data_type.get()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_type, - AdjustTimezone(src_map_type->key_type())); + NormalizeFileType(src_map_type->key_type())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item_type, - AdjustTimezone(src_map_type->item_type())); + NormalizeFileType(src_map_type->item_type())); return std::make_shared( src_map_type->key_field()->WithType(key_type), src_map_type->item_field()->WithType(item_type)); @@ -72,7 +69,7 @@ Result> ParquetTimestampConverter::AdjustTimezo case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr value_type, - AdjustTimezone(src_list_type->value_type())); + NormalizeFileType(src_list_type->value_type())); return arrow::list(src_list_type->value_field()->WithType(value_type)); } case arrow::Type::type::TIMESTAMP: { @@ -86,12 +83,25 @@ Result> ParquetTimestampConverter::AdjustTimezo } } -Result ParquetTimestampConverter::NeedCastArrayForTimestamp( +Result ParquetReadTypeAdapter::NeedsArrayConversion( const std::shared_ptr& src_data_type, const std::shared_ptr& target_data_type) { + return NeedsArrayConversionImpl(arrow::field("", src_data_type), + arrow::field("", target_data_type)); +} + +Result ParquetReadTypeAdapter::NeedsArrayConversionImpl( + const std::shared_ptr& src_field, + const std::shared_ptr& target_field) { + if (IsCompatibleBlob(src_field, target_field)) { + return false; + } + + const std::shared_ptr& src_data_type = src_field->type(); + const std::shared_ptr& target_data_type = target_field->type(); arrow::Type::type type = src_data_type->id(); if (type != target_data_type->id()) { - return Status::Invalid(fmt::format("src type {} and target type {} mismatch", + return Status::Invalid(fmt::format("source type {} and target type {} mismatch", src_data_type->ToString(), target_data_type->ToString())); } @@ -101,18 +111,13 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( auto* target_struct_type = checked_cast(target_data_type.get()); if (src_struct_type->num_fields() != target_struct_type->num_fields()) { return Status::Invalid( - fmt::format("src type {} and target type {} number of fields mismatch", + fmt::format("source type {} and target type {} number of fields mismatch", src_data_type->ToString(), target_data_type->ToString())); } for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { - const std::shared_ptr& src_field = src_struct_type->field(i); - const std::shared_ptr& target_field = target_struct_type->field(i); - if (IsAlreadyLogicalBlob(src_field, target_field)) { - continue; - } - PAIMON_ASSIGN_OR_RAISE( - bool need_cast, - NeedCastArrayForTimestamp(src_field->type(), target_field->type())); + PAIMON_ASSIGN_OR_RAISE(bool need_cast, + NeedsArrayConversionImpl(src_struct_type->field(i), + target_struct_type->field(i))); if (need_cast) { return true; } @@ -124,38 +129,54 @@ Result ParquetTimestampConverter::NeedCastArrayForTimestamp( auto* target_map_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( bool need_cast, - NeedCastArrayForTimestamp(src_map_type->key_type(), target_map_type->key_type())); + NeedsArrayConversionImpl(src_map_type->key_field(), target_map_type->key_field())); if (need_cast) { return true; } - PAIMON_ASSIGN_OR_RAISE( - need_cast, - NeedCastArrayForTimestamp(src_map_type->item_type(), target_map_type->item_type())); + PAIMON_ASSIGN_OR_RAISE(need_cast, + NeedsArrayConversionImpl(src_map_type->item_field(), + target_map_type->item_field())); return need_cast; } case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); auto* target_list_type = checked_cast(target_data_type.get()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, - NeedCastArrayForTimestamp(src_list_type->value_type(), - target_list_type->value_type())); - return need_cast; + return NeedsArrayConversionImpl(src_list_type->value_field(), + target_list_type->value_field()); } case arrow::Type::type::TIMESTAMP: { auto* src_ts_type = checked_cast(src_data_type.get()); auto* target_ts_type = checked_cast(target_data_type.get()); - if (src_ts_type->unit() != target_ts_type->unit() || - src_ts_type->timezone() != target_ts_type->timezone()) { - return true; - } - return false; + return src_ts_type->unit() != target_ts_type->unit() || + src_ts_type->timezone() != target_ts_type->timezone(); } default: return false; } } -Result> ParquetTimestampConverter::CastArrayForTimestamp( +Result> ParquetReadTypeAdapter::AdaptArray( + const std::shared_ptr& array, + const std::shared_ptr& target_data_type, + const std::shared_ptr& arrow_pool) { + PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedsArrayConversion(array->type(), target_data_type)); + if (!need_cast) { + return array; + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + AdaptArrayImpl(array, target_data_type, arrow_pool)); + PAIMON_ASSIGN_OR_RAISE(need_cast, NeedsArrayConversion(result->type(), target_data_type)); + if (need_cast) { + return Status::Invalid(fmt::format( + "unexpected: in parquet, after AdaptArray, output type {} does not match target " + "type {}", + result->type()->ToString(), target_data_type->ToString())); + } + return result; +} + +Result> ParquetReadTypeAdapter::AdaptArrayImpl( const std::shared_ptr& array, const std::shared_ptr& target_data_type, const std::shared_ptr& arrow_pool) { @@ -171,7 +192,7 @@ Result> ParquetTimestampConverter::CastArrayForTim const auto& field = struct_array->field(i); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr sub_array, - CastArrayForTimestamp(field, target_data_type->field(i)->type(), arrow_pool)); + AdaptArrayImpl(field, target_data_type->field(i)->type(), arrow_pool)); target_sub_arrays.push_back(sub_array); target_names.push_back(target_data_type->field(i)->name()); } @@ -187,10 +208,10 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* map_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr key_array, - CastArrayForTimestamp(map_array->keys(), map_type->key_type(), arrow_pool)); + AdaptArrayImpl(map_array->keys(), map_type->key_type(), arrow_pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr item_array, - CastArrayForTimestamp(map_array->items(), map_type->item_type(), arrow_pool)); + AdaptArrayImpl(map_array->items(), map_type->item_type(), arrow_pool)); return std::make_shared( arrow::map(key_array->type(), item_array->type()), map_array->length(), map_array->value_offsets(), key_array, item_array, map_array->null_bitmap(), @@ -201,7 +222,7 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* list_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr value_array, - CastArrayForTimestamp(list_array->values(), list_type->value_type(), arrow_pool)); + AdaptArrayImpl(list_array->values(), list_type->value_type(), arrow_pool)); return std::make_shared( arrow::list(value_array->type()), list_array->length(), list_array->value_offsets(), value_array, list_array->null_bitmap(), list_array->null_count(), @@ -213,8 +234,8 @@ Result> ParquetTimestampConverter::CastArrayForTim auto* ts_target_type = checked_cast(target_data_type.get()); if (src_type->unit() == arrow::TimeUnit::type::MILLI && ts_target_type->unit() == arrow::TimeUnit::type::SECOND) { - // parquet writer do not support second, and it cast second to milli. - // Therefore, in paimon file reader, we cast from milli to second. + // Parquet does not support second-precision timestamps, so the writer stores + // them as milliseconds. Convert them back to seconds for the Paimon read type. auto cast_executor = std::make_shared(); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr target_array, @@ -222,10 +243,9 @@ Result> ParquetTimestampConverter::CastArrayForTim return target_array; } if (src_type->timezone() != ts_target_type->timezone()) { - // 1. For nano type, parquet writer will write nano into int96 type, which does - // not contain any stats or zone info. Therefore in paimon file reader, we add - // zone info according to target type 2. For other precision, parquet reader - // will return UTC tz. Therefore, in paimon file reader, we add local zone info. + // INT96 carries no timezone, while other Parquet timestamps may be exposed as + // UTC. Attach the timezone required by the Paimon read type without changing + // the underlying values. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, ts_array->View(target_data_type)); return target_array; @@ -235,4 +255,5 @@ Result> ParquetTimestampConverter::CastArrayForTim return array; } } + } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.h b/src/paimon/format/parquet/parquet_read_type_adapter.h similarity index 59% rename from src/paimon/format/parquet/parquet_timestamp_converter.h rename to src/paimon/format/parquet/parquet_read_type_adapter.h index 2954bfe41..2f2960456 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter.h +++ b/src/paimon/format/parquet/parquet_read_type_adapter.h @@ -25,22 +25,34 @@ namespace paimon::parquet { -class ParquetTimestampConverter { +class ParquetReadTypeAdapter { public: - ParquetTimestampConverter() = delete; - ~ParquetTimestampConverter() = delete; + ParquetReadTypeAdapter() = delete; + ~ParquetReadTypeAdapter() = delete; - static Result> AdjustTimezone( + static Result> NormalizeFileType( const std::shared_ptr& src_data_type); - static Result NeedCastArrayForTimestamp( + static Result NeedsArrayConversion( const std::shared_ptr& src_data_type, const std::shared_ptr& target_data_type); - static Result> CastArrayForTimestamp( + static Result> AdaptArray( const std::shared_ptr& array, const std::shared_ptr& target_data_type, const std::shared_ptr& arrow_pool); + + private: + static Result> AdaptArrayImpl( + const std::shared_ptr& array, + const std::shared_ptr& target_data_type, + const std::shared_ptr& arrow_pool); + + static Result NeedsArrayConversionImpl(const std::shared_ptr& src_field, + const std::shared_ptr& target_field); + + static bool IsCompatibleBlob(const std::shared_ptr& src_field, + const std::shared_ptr& target_field); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp similarity index 76% rename from src/paimon/format/parquet/parquet_timestamp_converter_test.cpp rename to src/paimon/format/parquet/parquet_read_type_adapter_test.cpp index 784eecd05..17933cb82 100644 --- a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/format/parquet/parquet_timestamp_converter.h" +#include "paimon/format/parquet/parquet_read_type_adapter.h" #include @@ -31,7 +31,7 @@ namespace paimon::parquet::test { -TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { +TEST(ParquetReadTypeAdapterTest, TestNeedsArrayConversion) { { // single field need cast arrow::FieldVector fields = { @@ -40,10 +40,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { arrow::FieldVector target_fields = { arrow::field("f0", arrow::timestamp(arrow::TimeUnit::NANO, "UTC")), }; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in list need cast @@ -52,10 +52,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { }; arrow::FieldVector target_fields = { arrow::field("f2", arrow::list(arrow::timestamp(arrow::TimeUnit::SECOND)))}; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in map need cast @@ -66,10 +66,10 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { arrow::FieldVector target_fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::SECOND), arrow::timestamp(arrow::TimeUnit::NANO, "UTC")))}; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } { // field in struct need cast @@ -84,32 +84,32 @@ TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) { {arrow::field("f0", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f1", arrow::timestamp(arrow::TimeUnit::NANO, "UTC"))})), }; - ASSERT_OK_AND_ASSIGN(bool need_cast, - ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_OK_AND_ASSIGN(bool needs_conversion, + ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_(fields), arrow::struct_(target_fields))); - ASSERT_TRUE(need_cast); + ASSERT_TRUE(needs_conversion); } } -TEST(ParquetTimestampConverterTest, TestAlreadyLogicalBlobCompatibility) { +TEST(ParquetReadTypeAdapterTest, TestAlreadyLogicalBlobCompatibility) { std::shared_ptr restored_blob = arrow::field("blob", arrow::large_binary()); std::shared_ptr expected_physical_blob = BlobUtils::ToArrowField("blob")->WithType(arrow::binary()); - ASSERT_OK_AND_ASSIGN(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - arrow::struct_({restored_blob}), - arrow::struct_({expected_physical_blob}))); - ASSERT_FALSE(need_cast); + ASSERT_OK_AND_ASSIGN(bool needs_conversion, ParquetReadTypeAdapter::NeedsArrayConversion( + arrow::struct_({restored_blob}), + arrow::struct_({expected_physical_blob}))); + ASSERT_FALSE(needs_conversion); // Only a binary field carrying Paimon BLOB metadata is compatible with a large_binary field // already restored from ARROW:schema. A plain binary field must still be rejected. - ASSERT_NOK_WITH_MSG(ParquetTimestampConverter::NeedCastArrayForTimestamp( + ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::NeedsArrayConversion( arrow::struct_({restored_blob}), arrow::struct_({arrow::field("blob", arrow::binary())})), - "src type large_binary and target type binary mismatch"); + "source type large_binary and target type binary mismatch"); } -TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { +TEST(ParquetReadTypeAdapterTest, TestAdaptArray) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI), @@ -141,9 +141,9 @@ TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { .ValueOrDie()); std::shared_ptr pool = GetArrowPool(GetDefaultPool()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, - ParquetTimestampConverter::CastArrayForTimestamp( - array, arrow::struct_(target_fields), pool)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + ParquetReadTypeAdapter::AdaptArray(array, arrow::struct_(target_fields), pool)); auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(target_fields), R"([ @@ -155,7 +155,20 @@ TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) { ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); } -TEST(ParquetTimestampConverterTest, TestAdjustTimezone) { +TEST(ParquetReadTypeAdapterTest, TestAdaptArrayValidatesResultType) { + std::shared_ptr src_type = arrow::timestamp(arrow::TimeUnit::MICRO); + std::shared_ptr target_type = arrow::timestamp(arrow::TimeUnit::MILLI); + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(src_type, R"(["1970-01-01 00:00:01"])") + .ValueOrDie(); + std::shared_ptr pool = GetArrowPool(GetDefaultPool()); + + ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::AdaptArray(array, target_type, pool), + "after AdaptArray, output type timestamp[us] does not match target type " + "timestamp[ms]"); +} + +TEST(ParquetReadTypeAdapterTest, TestNormalizeFileType) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI), @@ -179,8 +192,8 @@ TEST(ParquetTimestampConverterTest, TestAdjustTimezone) { arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)), }; - ASSERT_OK_AND_ASSIGN(auto result_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(fields))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_type, + ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(fields))); ASSERT_TRUE(result_type->Equals(arrow::struct_(target_fields))); } } // namespace paimon::parquet::test diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 94d6c99fc..a55951503 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -419,6 +419,29 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) { ASSERT_EQ(inline_descriptor->Uri(), "file:///nonexistent/pypaimon-all-types-external-blob.bin"); ASSERT_EQ(inline_descriptor->Offset(), 7); ASSERT_EQ(inline_descriptor->Length(), 11); + + // Python and Java store MAP values in standard separate BLOB files. Validate + // resolved payloads, null values, and an empty map. Rust stores raw values inline in Parquet, + // which is not a compatible Paimon BLOB representation and is asserted separately below. + if (writer_prefix == "rust") { + return; + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr map_blob_result, + ReadTable(writer_prefix + "_map_blob_types", {"id", "f_map_blob"}, blob_value_options)); + std::shared_ptr map_blob_rows = GetOnlyStructChunk(map_blob_result); + ASSERT_TRUE(map_blob_rows); + ASSERT_EQ(map_blob_rows->length(), 3); + AssertFieldEqualsJson(map_blob_rows, "id", arrow::int32(), "[1, 2, 3]"); + AssertFieldEqualsJson(map_blob_rows, "f_map_blob", + arrow::map(arrow::utf8(), arrow::large_binary()), + R"([[ ["left", "blob-map-left"], ["right", null] ], null, []])"); +} + +TEST(PaimonReadCompatInteStandaloneTest, RejectsNonStandardRustMapBlob) { + ASSERT_NOK_WITH_MSG( + ReadTable("rust_map_blob_types", {"f_map_blob"}), + "Parquet does not support partial projection inside list/map: src map UnsupportedReadParams() { const std::vector read_cases = { - {"NestedBlob", "nested_blob_types", "f_array_blob", "Blob field must be a top-level field"}, + {"ArrayBlob", "array_blob_types", "f_array_blob", + "BLOB field must be a top-level field or the direct value of a top-level MAP field"}, {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md b/test/test_data/parquet/append_types_compatibility.db/README.md index 5e4e330be..a1ac373a3 100644 --- a/test/test_data/parquet/append_types_compatibility.db/README.md +++ b/test/test_data/parquet/append_types_compatibility.db/README.md @@ -19,7 +19,9 @@ # Paimon writer compatibility fixtures -The `append_types_compatibility.db` database contains equivalent append-only tables written by Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of types in that table; no individual table is claimed to contain every Paimon type. +The `append_types_compatibility.db` database contains append-only compatibility tables written by +Python, Rust, and Java Paimon. Table names use the writer as a prefix and describe the group of +types in that table; no individual table is claimed to contain every Paimon type. The writers and Parquet implementations are: @@ -29,11 +31,16 @@ The writers and Parquet implementations are: ## Table groups -Each writer prefix has four tables with the same schema and logical values: +Each writer prefix has five tables with the same declared schema and intended logical values: - `_types`: 43 columns and 3 rows covering integral and floating types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE, TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions, VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in every nullable column and row 3 exercises empty values. - `_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, and DOUBLE. -- `_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>. Paimon C++ rejects the table schema because ARRAY<BLOB> is unsupported. Schema validation happens before column projection, so the compatibility test explicitly checks this error instead of reading either field. +- `_array_blob_types`: Paimon C++ does not currently support ARRAY<BLOB>. +- `_map_blob_types`: MAP<STRING, BLOB>, including non-null, null, and empty map + values. Python and Java store these values in standard separate BLOB files, which Paimon C++ + can read. Rust stores the raw values inline as Parquet `binary`; this is not the Paimon BLOB + descriptor representation, cannot be read as BLOB by Java, and is intentionally rejected by + Paimon C++. - `_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++ currently rejects `TIME` while parsing the table schema. The tables are separated because Paimon C++ does not allow VECTOR in a data-evolution table, BLOB requires data evolution, and a schema-level incompatibility must not prevent compatible columns from being tested. @@ -42,4 +49,7 @@ The tables are separated because Paimon C++ does not allow VECTOR in a data-evol The `f_blob_descriptor` value intentionally points at a nonexistent external URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor and Parquet `ARROW:schema` path rather than external blob fetching. -Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types differ in these fixtures. The Python fixture records the physical Parquet `binary` column as Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test verifies that Paimon C++ can read both representations and preserve the descriptor value. +Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`, but the stored types +differ for `f_blob_descriptor`. The Python fixture records the physical Parquet `binary` column as +Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The compatibility test +verifies that Paimon C++ can read both representations and preserve the descriptor value. diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0 similarity index 88% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0 index f12e5f846..f6ca091da 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0 and b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0 similarity index 95% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0 index 034d68e17..4b7ac1f09 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0 and b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 new file mode 100644 index 000000000..b7ae3ef2c Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 new file mode 100644 index 000000000..0bce94c9f --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0 @@ -0,0 +1,28 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "f_array_blob", + "type" : { + "type" : "ARRAY", + "element" : "BLOB" + } + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "blob-field" : "f_array_blob", + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "avro", + "file.format" : "parquet" + }, + "timeMillis" : 1788859707631 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST rename to test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST diff --git a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..29a83ff0b --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "829d0558-0c81-4701-ba81-108a84c1de2e", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1", + "deltaManifestListSize" : 1115, + "commitUser" : "389ad12e-4421-4e4f-ad11-14284923cc57", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788859708818, + "totalRecordCount" : 6, + "deltaRecordCount" : 6, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet new file mode 100644 index 000000000..2d2cb3152 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 new file mode 100644 index 000000000..c803b1c9b Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0 similarity index 86% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0 index 071447210..655f831bf 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1 and b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1 new file mode 100644 index 000000000..9040aaeaa Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0 similarity index 69% rename from test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0 index 78945df1c..340c0ade0 100644 --- a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0 +++ b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0 @@ -7,13 +7,6 @@ "type" : "INT NOT NULL" }, { "id" : 1, - "name" : "f_array_blob", - "type" : { - "type" : "ARRAY", - "element" : "BLOB" - } - }, { - "id" : 2, "name" : "f_map_blob", "type" : { "type" : "MAP", @@ -21,16 +14,16 @@ "value" : "BLOB" } } ], - "highestFieldId" : 2, + "highestFieldId" : 1, "partitionKeys" : [ ], "primaryKeys" : [ ], "options" : { "bucket" : "-1", - "blob-field" : "f_array_blob,f_map_blob", + "blob-field" : "f_map_blob", "row-tracking.enabled" : "true", "data-evolution.enabled" : "true", "manifest.format" : "avro", "file.format" : "parquet" }, - "timeMillis" : 1788853436235 + "timeMillis" : 1788859708882 } \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST rename to test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST diff --git a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..edcebd32d --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "10cfbd5d-65f3-4ef3-b8b0-78c34e831369", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1", + "deltaManifestListSize" : 1115, + "commitUser" : "df37a7f9-0e78-4d8c-aa5c-13c284be6a37", + "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1788859708907, + "totalRecordCount" : 6, + "deltaRecordCount" : 6, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 deleted file mode 100644 index b98c7d14d..000000000 --- a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1 +++ /dev/null @@ -1,18 +0,0 @@ -{ - "version" : 3, - "uuid" : "89919eb0-8f08-4098-9002-d19680ca3a6d", - "id" : 1, - "schemaId" : 0, - "baseManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0", - "baseManifestListSize" : 1006, - "deltaManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1", - "deltaManifestListSize" : 1111, - "commitUser" : "1192f15e-31f1-42a4-b498-248a36a42985", - "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe", - "commitIdentifier" : 9223372036854775807, - "commitKind" : "APPEND", - "timeMillis" : 1788853436256, - "totalRecordCount" : 9, - "deltaRecordCount" : 9, - "nextRowId" : 3 -} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-043fd246-38df-40f7-9147-c1c416282ecc-0.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-043fd246-38df-40f7-9147-c1c416282ecc-0.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0 similarity index 87% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0 index eb9d186f6..3c001fe92 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0 and b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0 similarity index 96% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0 index 8608500a4..105bce895 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0 and b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1 similarity index 85% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 rename to test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1 index c85847cb0..cf8d9f9c5 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1 and b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0 new file mode 100644 index 000000000..9eff531ea --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0 @@ -0,0 +1,33 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT NOT NULL" + }, + { + "id": 1, + "name": "f_array_blob", + "type": { + "type": "ARRAY", + "element": "BLOB", + "nullable": true + } + } + ], + "highestFieldId": 1, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "parquet", + "manifest.format": "avro", + "row-tracking.enabled": "true", + "data-evolution.enabled": "true", + "blob-field": "f_array_blob" + }, + "comment": "Generated by pypaimon 2.0.0 with ARRAY.", + "timeMillis": 1788859561835 +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..ac1f6f1a0 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0", + "deltaManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "657b7800-ea12-4a26-8075-4b2fd4e20d51", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788859561849, + "nextRowId": 3, + "uuid": "43460238-f728-4ee1-8600-1f3d0c13234a" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet new file mode 100644 index 000000000..d733aa4d5 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob similarity index 100% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob rename to test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 new file mode 100644 index 000000000..f238c7da6 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0 new file mode 100644 index 000000000..97bbfb6e1 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1 new file mode 100644 index 000000000..ad23ae477 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 similarity index 61% rename from test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 rename to test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 index e311af948..08ed496fd 100644 --- a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0 +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0 @@ -9,15 +9,6 @@ }, { "id": 1, - "name": "f_array_blob", - "type": { - "type": "ARRAY", - "element": "BLOB", - "nullable": true - } - }, - { - "id": 2, "name": "f_map_blob", "type": { "type": "MAP", @@ -27,7 +18,7 @@ } } ], - "highestFieldId": 2, + "highestFieldId": 1, "partitionKeys": [], "primaryKeys": [], "options": { @@ -36,8 +27,8 @@ "manifest.format": "avro", "row-tracking.enabled": "true", "data-evolution.enabled": "true", - "blob-field": "f_array_blob,f_map_blob" + "blob-field": "f_map_blob" }, - "comment": "Generated by pypaimon 2.0.0 with ARRAY and MAP.", - "timeMillis": 1788513128455 + "comment": "Generated by pypaimon 2.0.0 with MAP.", + "timeMillis": 1788859561850 } \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..abae40fe9 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0", + "deltaManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1", + "totalRecordCount": 6, + "deltaRecordCount": 6, + "commitUser": "cadd033b-11dd-40e2-a759-0306ab9df091", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1788859561854, + "nextRowId": 3, + "uuid": "0279865a-f61e-41bf-8218-69adf804978f" +} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 deleted file mode 100644 index 8e955da73..000000000 --- a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1 +++ /dev/null @@ -1,15 +0,0 @@ -{ - "version": 3, - "id": 1, - "schemaId": 0, - "baseManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0", - "deltaManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1", - "totalRecordCount": 9, - "deltaRecordCount": 9, - "commitUser": "297eeaeb-5c62-4b3c-8f9e-950b09612bc3", - "commitIdentifier": 9223372036854775807, - "commitKind": "APPEND", - "timeMillis": 1788513128459, - "nextRowId": 3, - "uuid": "d2837b32-27c3-453e-8470-ccb9d176d3f0" -} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet new file mode 100644 index 000000000..f31d5eccf Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 new file mode 100644 index 000000000..0d557ef9d Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 new file mode 100644 index 000000000..760835cec Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1 similarity index 89% rename from test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 rename to test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1 index a01a7456a..c025a6df8 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0 and b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 new file mode 100644 index 000000000..c090dea09 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","row-tracking.enabled":"true","bucket":"-1","blob-field":"f_array_blob"},"comment":null,"timeMillis":1788860192967} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..51c4b5d99 --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-0","deltaManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-1","commitUser":"a8381718-0f79-46d1-92ec-2879a4cca6f8","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788860192978,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet new file mode 100644 index 000000000..08aadfe58 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 new file mode 100644 index 000000000..72f099052 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 new file mode 100644 index 000000000..68de8b163 Binary files /dev/null and b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1 similarity index 89% rename from test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 rename to test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1 index 5478f74d4..7050b7cf7 100644 Binary files a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1 and b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1 differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0 new file mode 100644 index 000000000..b6cd8894e --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0 @@ -0,0 +1 @@ +{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT NULL","value":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"blob-field":"f_map_blob","row-tracking.enabled":"true","file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","bucket":"-1"},"comment":null,"timeMillis":1788860192979} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1 new file mode 100644 index 000000000..d2926bd6b --- /dev/null +++ b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1 @@ -0,0 +1 @@ +{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0","deltaManifestList":"manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1","commitUser":"cb443c72-2132-4223-9af0-f39360413a6a","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788860192988,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet deleted file mode 100644 index 0383ee56b..000000000 Binary files a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet and /dev/null differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 deleted file mode 100644 index 286b07f38..000000000 Binary files a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 and /dev/null differ diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 deleted file mode 100644 index 66ceca0a3..000000000 --- a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0 +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}},{"id":2,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT NULL","value":"BLOB"}}],"highestFieldId":2,"partitionKeys":[],"primaryKeys":[],"options":{"manifest.format":"avro","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1","row-tracking.enabled":"true","blob-field":"f_array_blob,f_map_blob"},"comment":null,"timeMillis":1788858155021} \ No newline at end of file diff --git a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 deleted file mode 100644 index c24110af8..000000000 --- a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1 +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0","deltaManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1","commitUser":"ce789997-7558-44de-8068-c5d7e744dba9","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155032,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3} \ No newline at end of file