diff --git a/be/src/core/column/column_struct.cpp b/be/src/core/column/column_struct.cpp index fb15785df8bb2d..1a4455a2079025 100644 --- a/be/src/core/column/column_struct.cpp +++ b/be/src/core/column/column_struct.cpp @@ -120,6 +120,10 @@ void ColumnStruct::insert(const Field& x) { DCHECK_EQ(x.get_type(), PrimitiveType::TYPE_STRUCT); const auto& tuple = x.get(); const size_t tuple_size = columns.size(); + if (tuple.empty() && tuple_size != 0) { + insert_default(); + return; + } if (tuple.size() != tuple_size) { throw doris::Exception( ErrorCode::INTERNAL_ERROR, diff --git a/be/src/core/data_type_serde/complex_type_deserialize_util.h b/be/src/core/data_type_serde/complex_type_deserialize_util.h index b1320b4b073fc3..2792d7e8044c36 100644 --- a/be/src/core/data_type_serde/complex_type_deserialize_util.h +++ b/be/src/core/data_type_serde/complex_type_deserialize_util.h @@ -17,6 +17,7 @@ #pragma once +#include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_serde.h" #include "core/string_ref.h" @@ -80,13 +81,37 @@ struct ComplexTypeDeserializeUtil { static bool is_null_string(const StringRef& str) { if (str.size == 4) { - // null - return str.data[0] == 'n' && str.data[1] == 'u' && str.data[2] == 'l' && - str.data[3] == 'l'; + // SQL NULL literal is case-insensitive. + return (str.data[0] == 'n' || str.data[0] == 'N') && + (str.data[1] == 'u' || str.data[1] == 'U') && + (str.data[2] == 'l' || str.data[2] == 'L') && + (str.data[3] == 'l' || str.data[3] == 'L'); } return false; } + static Status process_field(const DataTypeSerDeSPtr& serde, StringRef str, Field& field) { + str = str.trim_whitespace(); + auto nullable_serde = std::dynamic_pointer_cast(serde); + if (is_null_string(str)) { + if (nullable_serde == nullptr) { + return Status::InvalidArgument( + "NULL default is not allowed for non-nullable complex field"); + } + field = Field::create_field(Null {}); + return Status::OK(); + } + auto str_without_quote = str.trim_quote(); + auto nested_serde = serde; + if (nullable_serde != nullptr) { + // Complex default values need strict nested conversion. DataTypeNullableSerDe + // converts nested parse failures to NULL, which would silently accept + // type-mismatched default literals. + nested_serde = nullable_serde->get_nested_serde(); + } + return nested_serde->from_fe_string(str_without_quote.to_string(), field); + } + template static Status process_column(const DataTypeSerDeSPtr& serde, IColumn& column, StringRef& str, const DataTypeSerDe::FormatOptions& options) { @@ -107,4 +132,4 @@ struct ComplexTypeDeserializeUtil { } }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index c3980f58821ee5..93d5582fc1cb78 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -593,6 +593,42 @@ Status DataTypeArraySerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeArraySerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '[') { + return Status::InvalidArgument("Array does not start with '[' character, found '{}'", + slice.to_string()); + } + if (slice.back() != ']') { + return Status::InvalidArgument("Array does not end with ']' character, found '{}'", + slice.to_string()); + } + + Array array; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter( + slice, [&](char c) { return c == options.collection_delim; }); + array.reserve(split_result.size()); + for (auto& item : split_result) { + Field item_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + nested_serde, item.element, item_field)); + array.push_back(std::move(item_field)); + } + } + } + field = Field::create_field(std::move(array)); + return Status::OK(); +} + Status DataTypeArraySerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index 31378a61081ede..d3d3413d3eb705 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -54,6 +54,8 @@ class DataTypeArraySerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index eb5e16f07700ad..8596ca3c6b0a18 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -756,6 +756,68 @@ Status DataTypeMapSerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeMapSerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '{') { + std::stringstream ss; + ss << slice.front() << '\''; + return Status::InvalidArgument("Map does not start with '{' character, found '" + ss.str()); + } + if (slice.back() != '}') { + std::stringstream ss; + ss << slice.back() << '\''; + return Status::InvalidArgument("Map does not end with '}' character, found '" + ss.str()); + } + + Array keys; + Array values; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) { + return c == options.map_key_delim || c == options.collection_delim; + }); + if (split_result.size() % 2 != 0) { + return Status::InvalidArgument("Map does not have even number of key-value pairs"); + } + keys.reserve(split_result.size() / 2); + values.reserve(split_result.size() / 2); + for (int i = 0; i < split_result.size(); i += 2) { + if (split_result[i].delimiter != options.map_key_delim) { + return Status::InvalidArgument( + "Map key-value pair does not have map key delimiter"); + } + if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Map key-value pair does not have collection delimiter"); + } + + Field key_field; + Field value_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + key_serde, split_result[i].element, key_field)); + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + value_serde, split_result[i + 1].element, value_field)); + keys.push_back(std::move(key_field)); + values.push_back(std::move(value_field)); + } + } + } + + Map map; + map.reserve(2); + map.push_back(Field::create_field(std::move(keys))); + map.push_back(Field::create_field(std::move(values))); + field = Field::create_field(std::move(map)); + return Status::OK(); +} + Status DataTypeMapSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index bd6560c053c102..44bd9634eeca5d 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -47,6 +47,8 @@ class DataTypeMapSerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index a295dc2799fc93..3ea1c221e8d743 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -734,6 +734,87 @@ Status DataTypeStructSerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeStructSerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '{') { + std::stringstream ss; + ss << slice.front() << '\''; + return Status::InvalidArgument("Struct does not start with '{' character, found '" + + ss.str()); + } + if (slice.back() != '}') { + std::stringstream ss; + ss << slice.back() << '\''; + return Status::InvalidArgument("Struct does not end with '}' character, found '" + + ss.str()); + } + + Struct struct_field; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) { + return c == options.map_key_delim || c == options.collection_delim; + }); + + const auto elem_size = elem_serdes_ptrs.size(); + std::vector field_values; + field_values.reserve(elem_size); + if (split_result.size() == elem_size) { + for (int i = 0; i < split_result.size(); i++) { + if (i != split_result.size() - 1 && + split_result[i].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Struct field value {} is not separated by collection_delim.", i); + } + field_values.push_back(split_result[i].element); + } + } else if (split_result.size() == 2 * elem_size) { + int field_pos = 0; + for (int i = 0; i < split_result.size(); i += 2) { + if (split_result[i].delimiter != options.map_key_delim) { + return Status::InvalidArgument( + "Struct name-value pair does not have map key delimiter"); + } + if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Struct name-value pair does not have collection delimiter"); + } + auto field_name = split_result[i].element.trim_quote(); + if (!field_name.eq(StringRef(elem_names[field_pos]))) { + return Status::InvalidArgument( + "Cannot find struct field name {} in schema.", + split_result[i].element.to_string()); + } + field_values.push_back(split_result[i + 1].element); + field_pos++; + } + } else { + return Status::InvalidArgument( + "Struct field number {} is not equal to schema field number {}.", + split_result.size(), elem_size); + } + + struct_field.reserve(elem_size); + for (int field_pos = 0; field_pos < elem_size; ++field_pos) { + Field elem_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + elem_serdes_ptrs[field_pos], field_values[field_pos], elem_field)); + struct_field.push_back(std::move(elem_field)); + } + } + } + + field = Field::create_field(std::move(struct_field)); + return Status::OK(); +} + Status DataTypeStructSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index 1cfa3ee6c74db7..6910fc7cddd4c7 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -47,6 +47,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp b/be/src/storage/iterator/vgeneric_iterators.cpp index fa75c4bb6382e3..c3fba014d5fe7e 100644 --- a/be/src/storage/iterator/vgeneric_iterators.cpp +++ b/be/src/storage/iterator/vgeneric_iterators.cpp @@ -48,8 +48,9 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) { const auto* column_desc = _schema.column(ordinal); auto unique_id = column_desc->unique_id(); if (_column_iterators_map.count(unique_id) < 1) { - RETURN_IF_ERROR(_segment->new_column_iterator( - *column_desc, &_column_iterators_map[unique_id], &opts)); + RETURN_IF_ERROR( + _segment->new_column_iterator(*column_desc, _schema.data_type(ordinal), + &_column_iterators_map[unique_id], &opts)); } _column_iterators.push_back(_column_iterators_map[unique_id].get()); } diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 6295091eb942ef..2cd9ab88db8bbb 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -44,6 +44,7 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_nullable.h" +#include "core/data_type/data_type_nullable.h" #include "exec/common/util.hpp" #include "exec/common/variant_util.h" #include "exprs/aggregate/aggregate_function.h" @@ -1528,8 +1529,11 @@ Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value) { if (!column_schema.is_nullable() || value.length() != 0) { - RETURN_IF_ERROR(column_schema.get_vec_type()->get_serde()->from_fe_string( - value, column_mapping->default_value)); + auto type = column_schema.get_vec_type(); + if (value.length() != 0) { + type = remove_nullable(type); + } + RETURN_IF_ERROR(type->get_serde()->from_fe_string(value, column_mapping->default_value)); } return Status::OK(); diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 842161f64d1838..f0cc7cec319d47 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -43,8 +43,11 @@ #include "core/column/column_vector.h" #include "core/custom_allocator.h" #include "core/data_type/data_type_agg_state.h" +#include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "core/decimal12.h" #include "core/string_ref.h" @@ -740,6 +743,92 @@ Status ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicat return Status::OK(); } +namespace { + +// Keep only the nested values of `field` that `read_type` materializes. FE nested pruning keeps +// the declared field order, never prunes MAP keys and leaves scalars untouched, so a STRUCT is +// rebuilt from the read field names while ARRAY items and MAP values recurse. +Status project_field_to_read_type(const DataTypePtr& declared_type, const DataTypePtr& read_type, + Field& field) { + switch (field.get_type()) { + case PrimitiveType::TYPE_ARRAY: { + const auto& declared_item = + assert_cast(remove_nullable(declared_type).get()) + ->get_nested_type(); + const auto& read_item = assert_cast(remove_nullable(read_type).get()) + ->get_nested_type(); + Array items; + items.reserve(field.get().size()); + for (const auto& item : field.get()) { + Field projected = item; + RETURN_IF_ERROR(project_field_to_read_type(declared_item, read_item, projected)); + items.push_back(std::move(projected)); + } + field = Field::create_field(std::move(items)); + return Status::OK(); + } + case PrimitiveType::TYPE_MAP: { + const auto& declared_value = + assert_cast(remove_nullable(declared_type).get()) + ->get_value_type(); + const auto& read_value = + assert_cast(remove_nullable(read_type).get())->get_value_type(); + // A MAP Field holds its keys and its values as two ARRAY Fields. + const auto& map = field.get(); + DORIS_CHECK_EQ(map.size(), 2); + Array values; + values.reserve(map[1].get().size()); + for (const auto& value : map[1].get()) { + Field projected = value; + RETURN_IF_ERROR(project_field_to_read_type(declared_value, read_value, projected)); + values.push_back(std::move(projected)); + } + field = Field::create_field( + Map {map[0], Field::create_field(std::move(values))}); + return Status::OK(); + } + case PrimitiveType::TYPE_STRUCT: { + const auto& tuple = field.get(); + // The `{}` default means every field takes its own default, which ColumnStruct::insert + // applies to whatever fields the destination keeps. + if (tuple.empty()) { + return Status::OK(); + } + const auto* declared_struct = + assert_cast(remove_nullable(declared_type).get()); + const auto* read_struct = + assert_cast(remove_nullable(read_type).get()); + DORIS_CHECK_EQ(tuple.size(), declared_struct->get_elements().size()); + Struct projected_tuple; + const size_t read_fields = read_struct->get_elements().size(); + projected_tuple.reserve(read_fields); + for (size_t i = 0; i < read_fields; ++i) { + const auto position = + declared_struct->try_get_position_by_name(read_struct->get_name_by_position(i)); + DORIS_CHECK(position.has_value()) + << "read struct field " << read_struct->get_name_by_position(i) + << " is not declared by " << declared_struct->get_name(); + Field projected = tuple[*position]; + RETURN_IF_ERROR(project_field_to_read_type(declared_struct->get_element(*position), + read_struct->get_element(i), projected)); + projected_tuple.push_back(std::move(projected)); + } + field = Field::create_field(std::move(projected_tuple)); + return Status::OK(); + } + default: + // A scalar or NULL has no nested value to prune. + return Status::OK(); + } +} + +} // namespace + +Status ConstantColumnIterator::project_to_read_type(const DataTypePtr& declared_type, + const DataTypePtr& read_type) { + return project_field_to_read_type(declared_type, read_type, _value); +} + Status ConstantColumnReader::match_condition(const AndBlockColumnPredicate* col_predicates, bool* matched) const { ZoneMap zone_map; diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index dfbd236c3d3248..74f2a867286ad7 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -962,6 +962,13 @@ class ConstantColumnIterator : public ColumnIterator { ConstantColumnIterator() = delete; explicit ConstantColumnIterator(Field value) : _value(std::move(value)) {} + // Nested column pruning materializes `read_type`, which keeps only the requested STRUCT fields + // of `declared_type` at any nesting depth. For example, after `ADD COLUMN s STRUCT DEFAULT '{7, "x"}'`, `struct_element(s, 'b')` reads old segments into a + // STRUCT column, so the default parsed with the declared type must become {"x"} + // before it is inserted; a file iterator drops its pruned sub-iterators instead. + Status project_to_read_type(const DataTypePtr& declared_type, const DataTypePtr& read_type); + Status seek_to_ordinal(ordinal_t ord_idx) override { _current_rowid = ord_idx; return Status::OK(); diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index af4ec1268a1d1b..0b103cd822313d 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -894,29 +894,22 @@ Status Segment::get_default_value_field(const TabletColumn& tablet_column, Field return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - if (default_value != "[]") { - return Status::NotSupported("Array default {} is unsupported", default_value); - } + // [] and {} need no nested SerDe, so an ARRAY/MAP TabletColumn built without sub-columns can + // still materialize an empty default. + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY && default_value == "[]") { *field = Field::create_field(Array {}); return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - return Status::NotSupported("STRUCT default type is unsupported"); - } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP) { - if (default_value != "{}") { - return Status::NotSupported("Map default {} is unsupported", default_value); - } + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP && default_value == "{}") { *field = Field::create_field(Map {Field::create_field(Array {}), Field::create_field(Array {})}); return Status::OK(); } - const auto serde = DataTypeFactory::instance() - .create_data_type(tablet_column.type(), tablet_column.precision(), - tablet_column.frac(), tablet_column.length()) - ->get_serde(); + // Any other non-null default, including a non-empty ARRAY/MAP or any STRUCT literal, needs the + // SerDe of the declared (possibly nested) type to parse recursively; a bare FieldType cannot + // describe nested item types. + const auto serde = remove_nullable(tablet_column.get_vec_type())->get_serde(); return serde->from_fe_string(default_value, *field); } @@ -935,7 +928,7 @@ Status Segment::new_constant_iterator(const TabletColumn& tablet_column, // in the new schema column c's cid == 2 // but in the old schema column b's cid == 2 // but they are not the same column -Status Segment::new_column_iterator(const TabletColumn& tablet_column, +Status Segment::new_column_iterator(const TabletColumn& tablet_column, const DataTypePtr& read_type, std::unique_ptr* iter, const StorageReadOptions* opt, const std::unordered_map* @@ -986,6 +979,15 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column, ? opt->predicate_access_paths.at(unique_id) : TColumnAccessPaths {}; + // A column absent from this segment was parsed from its default with the declared + // type, while the destination column is built from the pruned `read_type`. For + // example, `struct_element(s, 'b')` reads a light-added `s STRUCT + // DEFAULT '{7, "x"}'` into STRUCT, so the constant must become {"x"}. + if (auto* constant_iter = dynamic_cast(iter->get()); + constant_iter != nullptr) { + RETURN_IF_ERROR(constant_iter->project_to_read_type(tablet_column.get_vec_type(), + read_type)); + } // set column name to apply access paths. (*iter)->set_column_name(tablet_column.name()); RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths)); @@ -1435,7 +1437,8 @@ Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDesc MutableColumnPtr file_storage_column = storage_type->create_column(); if (iterator_hint == nullptr) { - RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR(new_column_iterator(column, slot->type(), &iterator_hint, + &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR( @@ -1458,8 +1461,8 @@ Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDesc DORIS_CHECK(variant_v2_type != nullptr); } if (iterator_hint == nullptr) { - RETURN_IF_ERROR( - new_column_iterator(read_column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR(new_column_iterator(read_column, slot->type(), &iterator_hint, + &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR(iterator_hint->read_by_rowids(row_ids.data(), row_ids.size(), result)); diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index dfdb88ab72cdd7..d3ec5c7ab7798a 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -155,8 +155,11 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd MOCK_FUNCTION uint32_t num_rows() const { return _num_rows; } + // `read_type` is the type the caller materializes the column into. It equals the declared + // type unless nested column pruning requested a subset of STRUCT fields, in which case a + // constant default parsed with the declared type is projected onto it. // if variant_sparse_column_cache is nullptr, means the sparse column cache is not used - Status new_column_iterator(const TabletColumn& tablet_column, + Status new_column_iterator(const TabletColumn& tablet_column, const DataTypePtr& read_type, std::unique_ptr* iter, const StorageReadOptions* opt, const std::unordered_map* variant_sparse_column_cache = nullptr); diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 76c57993424e03..f07c2afa7cd925 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -797,8 +797,8 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra continue; } std::unique_ptr iter; - RETURN_IF_ERROR( - _segment->new_column_iterator(*col, &iter, &_opts, &_variant_sparse_column_cache)); + RETURN_IF_ERROR(_segment->new_column_iterator(*col, _seek_schema->data_type(i), &iter, + &_opts, &_variant_sparse_column_cache)); ColumnIteratorOptions iter_opts { .use_page_cache = _opts.use_page_cache, .file_reader = _file_reader.get(), @@ -1904,9 +1904,9 @@ Status SegmentIterator::_init_column_iterators() { } if (_column_iterators[cid] == nullptr) { - RETURN_IF_ERROR(_segment->new_column_iterator(*_schema->column(cid), - &_column_iterators[cid], &_opts, - &_variant_sparse_column_cache)); + RETURN_IF_ERROR(_segment->new_column_iterator( + *_schema->column(cid), _schema->data_type(cid), &_column_iterators[cid], &_opts, + &_variant_sparse_column_cache)); ColumnIteratorOptions iter_opts { .use_page_cache = _opts.use_page_cache, // If the col is predicate column, then should read the last page to check diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index b3bbf336ab6de6..eeb77c9cf5360d 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -119,7 +119,8 @@ Status _init_segment_column_iterator(const BetaRowsetSharedPtr& rowset, if (input_io_ctx != nullptr) { opts.io_ctx = *input_io_ctx; } - RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, &opts)); + RETURN_IF_ERROR(segment->new_column_iterator(target_column, target_column.get_vec_type(), + column_iterator, &opts)); auto io_ctx = opts.io_ctx; io_ctx.reader_type = ReaderType::READER_QUERY; io_ctx.file_cache_stats = &stats.file_cache_stats; diff --git a/be/test/core/data_type_serde/data_type_from_string_test.cpp b/be/test/core/data_type_serde/data_type_from_string_test.cpp index 7eeff50fbbe3c4..12f2ab19c22467 100644 --- a/be/test/core/data_type_serde/data_type_from_string_test.cpp +++ b/be/test/core/data_type_serde/data_type_from_string_test.cpp @@ -35,6 +35,7 @@ #include "core/data_type/data_type_time.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/field.h" #include "core/string_ref.h" #include "gtest/gtest.h" #include "testutil/mock/mock_runtime_state.h" @@ -108,6 +109,133 @@ TEST_F(DataTypeSerDeFromStringTest, array) { } } +TEST_F(DataTypeSerDeFromStringTest, arrayFromFeString) { + auto type = std::make_shared( + std::make_shared(std::make_shared())); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"([1, NULL, nUlL, 3])", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_ARRAY); + const auto& array = field.get(); + ASSERT_EQ(array.size(), 4); + EXPECT_EQ(array[0].get(), 1); + EXPECT_TRUE(array[1].is_null()); + EXPECT_TRUE(array[2].is_null()); + EXPECT_EQ(array[3].get(), 3); + + EXPECT_FALSE(serde->from_fe_string(R"({"not_array": 1})", field).ok()); +} + +TEST_F(DataTypeSerDeFromStringTest, mapFromFeString) { + auto type = std::make_shared( + std::make_shared(), + std::make_shared(std::make_shared())); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"({"a": 1, "b": NULL, "c": nUlL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_MAP); + const auto& map = field.get(); + ASSERT_EQ(map.size(), 2); + ASSERT_EQ(map[0].get_type(), TYPE_ARRAY); + ASSERT_EQ(map[1].get_type(), TYPE_ARRAY); + + const auto& keys = map[0].get(); + const auto& values = map[1].get(); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(keys[0].get(), "a"); + EXPECT_EQ(values[0].get(), 1); + EXPECT_EQ(keys[1].get(), "b"); + EXPECT_TRUE(values[1].is_null()); + EXPECT_EQ(keys[2].get(), "c"); + EXPECT_TRUE(values[2].is_null()); + + EXPECT_FALSE(serde->from_fe_string(R"({"a": 1, "b"})", field).ok()); +} + +TEST_F(DataTypeSerDeFromStringTest, structFromFeString) { + DataTypes elems {std::make_shared(std::make_shared()), + std::make_shared(), + std::make_shared(std::make_shared())}; + Strings names {"f1", "f2", "f3"}; + auto type = std::make_shared(elems, names); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"({NULL, "x", nUlL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_STRUCT); + const auto& tuple_values = field.get(); + ASSERT_EQ(tuple_values.size(), 3); + EXPECT_TRUE(tuple_values[0].is_null()); + EXPECT_EQ(tuple_values[1].get(), "x"); + EXPECT_TRUE(tuple_values[2].is_null()); + + ASSERT_TRUE(serde->from_fe_string(R"({"f1": 7, "f2": "y", "f3": NULL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_STRUCT); + const auto& named_values = field.get(); + ASSERT_EQ(named_values.size(), 3); + EXPECT_EQ(named_values[0].get(), 7); + EXPECT_EQ(named_values[1].get(), "y"); + EXPECT_TRUE(named_values[2].is_null()); + + EXPECT_FALSE(serde->from_fe_string(R"({1, 2})", field).ok()); + EXPECT_FALSE(serde->from_fe_string(R"({"f2": 1, "f1": "x", "f3": 3})", field).ok()); + + DataTypes non_nullable_elems {std::make_shared()}; + Strings non_nullable_names {"f1"}; + auto non_nullable_type = + std::make_shared(non_nullable_elems, non_nullable_names); + auto non_nullable_serde = non_nullable_type->get_serde(); + auto status = non_nullable_serde->from_fe_string(R"({NULL})", field); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("NULL default is not allowed") != std::string::npos); +} + +// FE stores complex defaults in a canonical text: unquoted numbers (bool as 1/0), double quoted +// strings/dates/ips and nested brackets. These cases pin down what BE must accept from that text. +TEST_F(DataTypeSerDeFromStringTest, canonicalComplexDefaultFromFeString) { + auto check = [](const DataTypePtr& type, const std::string& text, const std::string& expected) { + Field field; + auto st = type->get_serde()->from_fe_string(text, field); + ASSERT_TRUE(st.ok()) << text << ": " << st.to_string(); + auto column = type->create_column(); + column->insert(field); + EXPECT_EQ(type->to_string(*column, 0), expected) << text; + }; + auto nullable = [](DataTypePtr type) { return std::make_shared(type); }; + + check(std::make_shared(nullable(std::make_shared())), + R"(["it's", "x,y", "[z]", "{k:v}", "null", "", NULL])", + R"(["it's", "x,y", "[z]", "{k:v}", "null", "", null])"); + check(std::make_shared(nullable(std::make_shared())), + R"(["2024-01-01"])", R"(["2024-01-01"])"); + check(std::make_shared(nullable(std::make_shared(0))), + R"(["2024-01-01 10:20:30"])", R"(["2024-01-01 10:20:30"])"); + check(std::make_shared(nullable(std::make_shared(3))), + R"(["2024-01-01 10:20:30.123"])", R"(["2024-01-01 10:20:30.123"])"); + check(std::make_shared(nullable(std::make_shared())), "[1, 0]", + "[1, 0]"); + check(std::make_shared(nullable(std::make_shared())), + "[1.0E10, 1.5]", "[10000000000, 1.5]"); + check(std::make_shared(nullable(std::make_shared(10, 2))), + "[1.23, 1000]", "[1.23, 1000.00]"); + check(std::make_shared(nullable(std::make_shared())), + R"(["127.0.0.1"])", R"(["127.0.0.1"])"); + check(std::make_shared(std::make_shared(), + nullable(std::make_shared( + nullable(std::make_shared())))), + R"({"a":[1, 2], "b":[], "c":NULL})", R"({"a":[1, 2], "b":[], "c":null})"); + check(std::make_shared( + DataTypes {nullable(std::make_shared()), + nullable(std::make_shared( + std::make_shared(), + nullable(std::make_shared())))}, + Strings {"f1", "f2"}), + R"({7, {"x":1}})", R"({"f1":7, "f2":{"x":1}})"); +} + TEST_F(DataTypeSerDeFromStringTest, bitmap) { auto type = std::make_shared(); auto serde = type->get_serde(); @@ -673,4 +801,4 @@ TEST_F(DataTypeSerDeFromStringTest, timeTest) { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/storage/index/index_builder_test.cpp b/be/test/storage/index/index_builder_test.cpp index 3838529874bac6..9159522395199e 100644 --- a/be/test/storage/index/index_builder_test.cpp +++ b/be/test/storage/index/index_builder_test.cpp @@ -729,7 +729,9 @@ class IndexBuilderCommitTsoTest : public IndexBuilderTest, options.commit_tso = rowset->rowset_meta()->commit_tso(); options.io_ctx.reader_type = ReaderType::READER_QUERY; segment_v2::ColumnIteratorUPtr iter; - st = segment->new_column_iterator(rowset->tablet_schema()->column(2), &iter, &options); + st = segment->new_column_iterator(rowset->tablet_schema()->column(2), + rowset->tablet_schema()->column(2).get_vec_type(), &iter, + &options); // Published singleton rowsets with absent or [-1, -1] TSOs must reject reads, // even though index build/drop must preserve that metadata unchanged. if (!tso.has_value() || tso->end_tso() == -1) { diff --git a/be/test/storage/segment/constant_column_iterator_test.cpp b/be/test/storage/segment/constant_column_iterator_test.cpp index 27e5ef383ec944..77ea461f897dd0 100644 --- a/be/test/storage/segment/constant_column_iterator_test.cpp +++ b/be/test/storage/segment/constant_column_iterator_test.cpp @@ -35,6 +35,11 @@ #include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" #include "core/field.h" #include "core/value/bitmap_value.h" #include "core/value/hll.h" @@ -973,9 +978,10 @@ TEST_F(ConstantColumnIteratorTest, StructNullDefaultAndRuntimeConstantMaterializ EXPECT_EQ("doris", materialized_struct->get_column(1).get_data_at(1).to_string()); } -TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { - // Case: schema defaults currently support only [] and {} for complex non-null values. Reject - // non-empty text forms explicitly instead of silently synthesizing a different value. +TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreParsed) { + // Case: a non-empty ARRAY/MAP/STRUCT literal default is parsed through the declared (nested) + // type's SerDe instead of being rejected, so `[1]`, `{"k":7}`, and a positional struct literal + // all materialize the matching typed Field. TabletColumn int_child; int_child.set_type(FieldType::OLAP_FIELD_TYPE_INT); int_child.set_is_nullable(false); @@ -988,9 +994,11 @@ TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { array_column.set_is_nullable(false); array_column.set_default_value("[1]"); array_column.add_sub_column(int_child); - Field field; - EXPECT_TRUE(Segment::get_default_value_field(array_column, &field) - .is()); + Field array_field; + ASSERT_TRUE(Segment::get_default_value_field(array_column, &array_field).ok()); + const auto& array = array_field.get(); + ASSERT_EQ(1, array.size()); + EXPECT_EQ(1, array[0].get()); TabletColumn map_column; map_column.set_type(FieldType::OLAP_FIELD_TYPE_MAP); @@ -998,17 +1006,162 @@ TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { map_column.set_default_value("{\"k\":7}"); map_column.add_sub_column(string_child); map_column.add_sub_column(int_child); - EXPECT_TRUE(Segment::get_default_value_field(map_column, &field) - .is()); + Field map_field; + ASSERT_TRUE(Segment::get_default_value_field(map_column, &map_field).ok()); + const auto& map = map_field.get(); + ASSERT_EQ(2, map.size()); + const auto& keys = map[0].get(); + const auto& values = map[1].get(); + ASSERT_EQ(1, keys.size()); + ASSERT_EQ(1, values.size()); + EXPECT_EQ("k", std::string_view(keys[0].get())); + EXPECT_EQ(7, values[0].get()); + + // Positional struct literal: int_child/string_child have no declared name, so the literal is + // matched by field position instead of by name. + TabletColumn struct_column; + struct_column.set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); + struct_column.set_is_nullable(false); + struct_column.set_default_value("{99,\"doris\"}"); + struct_column.add_sub_column(int_child); + struct_column.add_sub_column(string_child); + Field struct_field; + ASSERT_TRUE(Segment::get_default_value_field(struct_column, &struct_field).ok()); + const auto& struct_value = struct_field.get(); + ASSERT_EQ(2, struct_value.size()); + EXPECT_EQ(99, struct_value[0].get()); + EXPECT_EQ("doris", std::string_view(struct_value[1].get())); +} + +TEST_F(ConstantColumnIteratorTest, ComplexDefaultIsProjectedToPrunedReadType) { + // Case: nested column pruning reads `struct_element(s, 'text')` of a light-added + // STRUCT DEFAULT '{99, "doris"}' into a STRUCT column. + // The constant parsed with the declared type must keep only the read fields, also for STRUCT + // items of an ARRAY and STRUCT values of a MAP, while the `{}` default stays empty so the + // destination fills its own defaults. + TabletColumn int_child; + int_child.set_name("number"); + int_child.set_type(FieldType::OLAP_FIELD_TYPE_INT); + int_child.set_is_nullable(false); + + TabletColumn string_child; + string_child.set_name("text"); + string_child.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + string_child.set_is_nullable(false); TabletColumn struct_column; + struct_column.set_name("s"); struct_column.set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); struct_column.set_is_nullable(false); - struct_column.set_default_value("(99, doris)"); + struct_column.set_default_value("{99,\"doris\"}"); struct_column.add_sub_column(int_child); struct_column.add_sub_column(string_child); - EXPECT_TRUE(Segment::get_default_value_field(struct_column, &field) - .is()); + // FE prunes the slot type to the accessed fields, in declared order, keeping nullability. + DataTypePtr read_struct = std::make_shared( + DataTypes {std::make_shared()}, Strings {"text"}); + + Field struct_default; + auto st = Segment::get_default_value_field(struct_column, &struct_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator struct_iterator(struct_default); + st = struct_iterator.project_to_read_type(struct_column.get_vec_type(), read_struct); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr struct_dst = read_struct->create_column(); + size_t rows = 2; + bool has_null = true; + st = struct_iterator.next_batch(&rows, struct_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_struct = assert_cast(struct_dst.get()); + ASSERT_EQ(1, materialized_struct->tuple_size()); + ASSERT_EQ(2, materialized_struct->size()); + EXPECT_EQ("doris", materialized_struct->get_column(0).get_data_at(0).to_string()); + EXPECT_EQ("doris", materialized_struct->get_column(0).get_data_at(1).to_string()); + + // Projection to the declared type itself keeps every field. + ConstantColumnIterator full_iterator(struct_default); + st = full_iterator.project_to_read_type(struct_column.get_vec_type(), + struct_column.get_vec_type()); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr full_dst = struct_column.get_vec_type()->create_column(); + rows = 1; + st = full_iterator.next_batch(&rows, full_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* full_struct = assert_cast(full_dst.get()); + ASSERT_EQ(2, full_struct->tuple_size()); + EXPECT_EQ(99, assert_cast(full_struct->get_column(0)).get_element(0)); + EXPECT_EQ("doris", full_struct->get_column(1).get_data_at(0).to_string()); + + // ARRAY: every item is projected. + TabletColumn array_column; + array_column.set_name("arr"); + array_column.set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); + array_column.set_is_nullable(false); + array_column.set_default_value("[{1,\"p\"},{2,\"q\"}]"); + array_column.add_sub_column(struct_column); + DataTypePtr read_array = std::make_shared(make_nullable(read_struct)); + Field array_default; + st = Segment::get_default_value_field(array_column, &array_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator array_iterator(array_default); + st = array_iterator.project_to_read_type(array_column.get_vec_type(), read_array); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr array_dst = read_array->create_column(); + rows = 1; + st = array_iterator.next_batch(&rows, array_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_array = assert_cast(array_dst.get()); + ASSERT_EQ(2, materialized_array->size_at(0)); + const auto& array_items = assert_cast( + assert_cast(materialized_array->get_data()).get_nested_column()); + ASSERT_EQ(1, array_items.tuple_size()); + EXPECT_EQ("p", array_items.get_column(0).get_data_at(0).to_string()); + EXPECT_EQ("q", array_items.get_column(0).get_data_at(1).to_string()); + + // MAP: keys are never pruned, every value is projected. + TabletColumn map_column; + map_column.set_name("m"); + map_column.set_type(FieldType::OLAP_FIELD_TYPE_MAP); + map_column.set_is_nullable(false); + map_column.set_default_value("{\"k\":{3,\"r\"}}"); + map_column.add_sub_column(string_child); + map_column.add_sub_column(struct_column); + DataTypePtr read_map = std::make_shared( + make_nullable(std::make_shared()), make_nullable(read_struct)); + Field map_default; + st = Segment::get_default_value_field(map_column, &map_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator map_iterator(map_default); + st = map_iterator.project_to_read_type(map_column.get_vec_type(), read_map); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr map_dst = read_map->create_column(); + rows = 1; + st = map_iterator.next_batch(&rows, map_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_map = assert_cast(map_dst.get()); + ASSERT_EQ(1, materialized_map->size_at(0)); + EXPECT_EQ("k", materialized_map->get_keys().get_data_at(0).to_string()); + const auto& map_values = assert_cast( + assert_cast(materialized_map->get_values()).get_nested_column()); + ASSERT_EQ(1, map_values.tuple_size()); + EXPECT_EQ("r", map_values.get_column(0).get_data_at(0).to_string()); + + // `{}` means every field takes its own default, so the pruned destination fills itself. + struct_column.set_default_value("{}"); + Field empty_default; + st = Segment::get_default_value_field(struct_column, &empty_default); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(empty_default.get().empty()); + ConstantColumnIterator empty_iterator(empty_default); + st = empty_iterator.project_to_read_type(struct_column.get_vec_type(), read_struct); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr empty_dst = read_struct->create_column(); + rows = 2; + st = empty_iterator.next_batch(&rows, empty_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* empty_struct = assert_cast(empty_dst.get()); + ASSERT_EQ(1, empty_struct->tuple_size()); + ASSERT_EQ(2, empty_struct->size()); + EXPECT_EQ("", empty_struct->get_column(0).get_data_at(1).to_string()); } TEST_F(ConstantColumnIteratorTest, VariantConstantMaterializesSubcolumnsAndZoneMap) { diff --git a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp index ed490410d1a632..e90f80ee8ff120 100644 --- a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp +++ b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp @@ -322,7 +322,9 @@ class SegmentIteratorExprZonemapTest : public testing::Test { void read_column(const std::shared_ptr& segment, int32_t cid, const StorageReadOptions& read_options, MutableColumnPtr* dst) { ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(_tablet_schema->column(cid), &iter, &read_options); + auto st = segment->new_column_iterator(_tablet_schema->column(cid), + _tablet_schema->column(cid).get_vec_type(), &iter, + &read_options); ASSERT_TRUE(st.ok()) << st; ASSERT_NE(nullptr, iter); @@ -351,7 +353,9 @@ class SegmentIteratorExprZonemapTest : public testing::Test { void expect_missing_commit_tso_error(const std::shared_ptr& segment, int32_t cid, const StorageReadOptions& read_options) { ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(_tablet_schema->column(cid), &iter, &read_options); + auto st = segment->new_column_iterator(_tablet_schema->column(cid), + _tablet_schema->column(cid).get_vec_type(), &iter, + &read_options); EXPECT_TRUE(st.is()) << st; EXPECT_NE(st.to_string().find("requires a valid commit tso"), std::string::npos) << st; } @@ -887,7 +891,7 @@ TEST_F(SegmentIteratorExprZonemapTest, MissingVariantDocBucketsDefaultToNull) { auto column = variant_util::create_doc_value_compaction_column(_tablet_schema->column(1), bucket); ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(column, &iter, &read_options); + auto st = segment->new_column_iterator(column, column.get_vec_type(), &iter, &read_options); ASSERT_TRUE(st.ok()) << st; ColumnIteratorOptions opts; opts.stats = &_stats; @@ -948,7 +952,8 @@ TEST_F(SegmentIteratorExprZonemapTest, MissingVariantSeparatesValueAndPhysicalRe // Reading a generated child path must use the missing root's logical NULL value. It must not // attempt to cast the ConstantColumnReader to VariantColumnReader or search sparse metadata. ColumnIteratorUPtr path_iterator; - st = segment->new_column_iterator(added_path, &path_iterator, &read_options); + st = segment->new_column_iterator(added_path, declared_path_type, &path_iterator, + &read_options); ASSERT_TRUE(st.ok()) << st; ASSERT_NE(nullptr, path_iterator); ColumnIteratorOptions iterator_options; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index 6272cfc0a3be4b..e0b4ecfe43c3fb 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -890,6 +890,18 @@ public String toSql() { return toSql(false, false); } + /** + * The DDL parser keeps the text between the quotes of a default value verbatim, so a value that + * contains double quotes (complex type defaults store nested strings as "...") is wrapped in single + * quotes to let SHOW CREATE TABLE output be replayed by CREATE TABLE LIKE. + */ + private static String quoteDefaultValue(String value) { + if (value.indexOf('"') >= 0 && value.indexOf('\'') < 0) { + return "'" + value + "'"; + } + return "\"" + value + "\""; + } + public String toSql(boolean isUniqueTable) { return toSql(isUniqueTable, false); } @@ -930,7 +942,7 @@ public String toSql(boolean isUniqueTable, boolean isCompatible, boolean useGene if (defaultValueExprDef != null) { sb.append(" DEFAULT ").append(defaultValue).append(""); } else { - sb.append(" DEFAULT \"").append(defaultValue).append("\""); + sb.append(" DEFAULT ").append(quoteDefaultValue(defaultValue)); } } if ((getDataType() == PrimitiveType.BITMAP) && defaultValue != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java index 7c0c4b2fd97221..0022b4d206cf99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; @@ -49,6 +50,8 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.plans.commands.info.ComplexTypeDefaultValue; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileFormatType; @@ -599,10 +602,23 @@ private static void validateDefaultValue(List args, Column column) throw } if (args.get(0) != null) { - ColumnDef.validateDefaultValue(column.getOriginType(), args.get(0), column.getDefaultValueExprDef()); + validateColumnValue(column, args.get(0)); } } + private static void validateColumnValue(Column column, String value) throws AnalysisException { + Type type = column.getOriginType(); + if (type.isComplexType()) { + try { + ComplexTypeDefaultValue.canonicalize(DataType.fromCatalogType(type), value); + } catch (org.apache.doris.nereids.exceptions.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + return; + } + ColumnDef.validateDefaultValue(type, value, column.getDefaultValueExprDef()); + } + private static void validateMd5sum(List args, Map columnNameMap) throws AnalysisException { for (int i = 0; i < args.size(); ++i) { String argColumn = args.get(i); @@ -634,7 +650,7 @@ private static void validateReplaceValue(List args, Column column) throw } if (replaceValue != null) { - ColumnDef.validateDefaultValue(column.getOriginType(), replaceValue, column.getDefaultValueExprDef()); + validateColumnValue(column, replaceValue); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java index 8b22fca5409026..633c4e874931c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java @@ -43,7 +43,9 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.commands.info.ComplexTypeDefaultValue; import org.apache.doris.nereids.trees.plans.commands.info.DefaultValue; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VarcharType; @@ -483,7 +485,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp exprs.add(new Not(new IsNull(slot))); exprs.add(slot); if (funcExpr.children().size() > 1) { - exprs.add(funcExpr.child(1)); + exprs.add(canonicalizeComplexValue(column, funcExpr.child(1))); } else { if (column.getDefaultValue() != null) { String exprSql = column.getDefaultValueSql(); @@ -503,7 +505,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp innerIfExprs.add(new Not(new EqualTo(slot, funcExpr.child(0)))); innerIfExprs.add(slot); if (funcExpr.children().size() > 1) { - innerIfExprs.add(funcExpr.child(1)); + innerIfExprs.add(canonicalizeComplexValue(column, funcExpr.child(1))); } else { if (column.getDefaultValue() != null) { String exprSql = column.getDefaultValueSql(); @@ -571,7 +573,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp return unixTimeFunc; } else if (funcName.equalsIgnoreCase("default_value")) { - return funcExpr.child(0); + return canonicalizeComplexValue(column, funcExpr.child(0)); } else if (funcName.equalsIgnoreCase("now")) { UnboundFunction newFunc = new UnboundFunction("now", Lists.newArrayList()); return newFunc; @@ -581,4 +583,18 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp } return originExpr; } + + /** + * The explicit value of a default_value or replace_value mapping on an ARRAY/MAP/STRUCT column was + * validated by {@link NereidsDataDescription#validateMappingFunction} as a SQL literal, but BE casts + * the mapping expression as a string. Replace it with the canonical text so that both agree, e.g. + * {@code '[1e3]'} for an array of INT is written as {@code [1000]} instead of a NULL element. + */ + private static Expression canonicalizeComplexValue(Column column, Expression value) { + if (!column.getOriginType().isComplexType() || !(value instanceof StringLikeLiteral)) { + return value; + } + return new StringLiteral(ComplexTypeDefaultValue.canonicalize( + DataType.fromCatalogType(column.getOriginType()), ((StringLikeLiteral) value).getStringValue())); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 7a30b276da158f..1ea4a4986f1305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -304,10 +304,21 @@ public List> parseMultiple(String sql, } public Expression parseExpression(String expression) { + return parseExpression(expression, null); + } + + /** + * parse an expression with the given logical plan builder. + * + * @param expression expression string + * @param logicalPlanBuilder logical plan builder, null for the default one + * @return expression + */ + public Expression parseExpression(String expression, @Nullable LogicalPlanBuilder logicalPlanBuilder) { if (isValidUnquotedIdentifier(expression)) { return new UnboundSlot(expression); } - return parse(expression, DorisParser::expressionWithEof); + return parse(expression, logicalPlanBuilder, DorisParser::expressionWithEof); } /** Return whether the text can be emitted as an unquoted Nereids identifier. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index ae06f269cca17b..f7511126dcc9ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -538,17 +538,11 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); - } else if (type.isArrayType() && defaultValue.isPresent() && isOlap - && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get() - .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { - throw new AnalysisException("Array type column default value only support null or " - + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); } else { validateComplexTypeDefaultValue(); } - if (!isNullable && defaultValue.isPresent() - && defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { + if (!isNullable && hasNullDefaultValue()) { throw new AnalysisException( "Can not set null default value to non nullable column: " + name); } @@ -628,20 +622,28 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c /** * Validate non-null defaults for complex types before connector-specific validation. + * ARRAY, MAP and STRUCT defaults are rewritten into the canonical literal text that BE parses. */ public void validateComplexTypeDefaultValue() throws AnalysisException { - if (!defaultValue.isPresent() || defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { + if (!hasNonNullDefaultValue()) { return; } - if (type.isMapType()) { - throw new AnalysisException("Map type column default value just support null"); - } else if (type.isStructType()) { - throw new AnalysisException("Struct type column default value just support null"); + if (type.isArrayType() || type.isMapType() || type.isStructType()) { + defaultValue = Optional.of(new DefaultValue( + ComplexTypeDefaultValue.canonicalize(type, defaultValue.get().getValue()))); } else if (type.isJsonType() || type.isVariantType()) { - throw new AnalysisException("Json or Variant type column default value just support null"); + throw new AnalysisException("Json or Variant type column default value only supports DEFAULT NULL"); } } + private boolean hasNonNullDefaultValue() { + return defaultValue.isPresent() && defaultValue.get().getValue() != null; + } + + private boolean hasNullDefaultValue() { + return defaultValue.isPresent() && defaultValue.get().getValue() == null; + } + /** * translate to catalog create table stmt */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java new file mode 100644 index 00000000000000..9398e8a4fc7945 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.nereids.DorisParser.ArrayLiteralContext; +import org.apache.doris.nereids.DorisParser.ConstantContext; +import org.apache.doris.nereids.DorisParser.MapLiteralContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.LogicalPlanBuilder; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.coercion.CharacterType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.ParserRuleContext; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; + +/** + * Validates a non-null ARRAY/MAP/STRUCT column default and rewrites it into a canonical literal text. + * + *

The user supplied text is parsed as a SQL literal whose nested values are kept exactly as written + * (see {@link LiteralBuilder}), and every nested value is then cast to the declared nested type, so a + * type mismatch is rejected at DDL time instead of when old rows are read. The + * canonical text only contains plain nested values (unquoted numbers, double quoted strings, nested + * brackets and lower case null). BE parses the stored text in three places that must agree: the + * complex SerDe {@code from_fe_string} used by the default value iterator and schema change, the + * string-to-complex cast used when INSERT fills an unmentioned column, and the JSON parser + * {@code default_from_string} that fills the omitted columns of a partial update, which only + * recognizes a lower case nested null. None of them decodes escape sequences, and the DDL parser + * keeps the text of a default value verbatim when SHOW CREATE TABLE output is replayed, so string + * values containing quotes or backslashes are rejected instead of stored. + */ +public class ComplexTypeDefaultValue { + private ComplexTypeDefaultValue() { + } + + /** + * Validate the default literal of a complex column and return its canonical text. + */ + public static String canonicalize(DataType type, String defaultValue) throws AnalysisException { + Preconditions.checkArgument(type.isArrayType() || type.isMapType() || type.isStructType(), + "%s is not a complex type", type); + Expression expression; + try { + expression = new NereidsParser().parseExpression(defaultValue, new LiteralBuilder()); + } catch (Exception e) { + throw literalShapeException(type); + } + if (!hasLiteralShape(expression, type)) { + throw literalShapeException(type); + } + try { + return render((Literal) expression, type); + } catch (AnalysisException e) { + throw new AnalysisException(String.format("Invalid default value '%s' for %s column: %s", + defaultValue, type.toSql(), e.getMessage()), e); + } + } + + private static boolean hasLiteralShape(Expression expression, DataType type) { + if (type.isArrayType()) { + return expression instanceof ArrayLiteral; + } + if (type.isMapType()) { + return expression instanceof MapLiteral; + } + // `{}` parses as an empty map literal and means every struct field takes its own default. + return expression instanceof StructLiteral + || (expression instanceof MapLiteral && ((MapLiteral) expression).getValue().isEmpty()); + } + + private static AnalysisException literalShapeException(DataType type) { + String literalKind = type.isArrayType() ? "array" : type.isMapType() ? "map" : "struct"; + return new AnalysisException(String.format("%s type column default value only supports %s literals" + + " or DEFAULT NULL", capitalize(literalKind), literalKind)); + } + + private static String capitalize(String value) { + return Character.toUpperCase(value.charAt(0)) + value.substring(1); + } + + private static String render(Literal literal, DataType type) throws AnalysisException { + if (literal instanceof NullLiteral) { + return "null"; + } + if (type.isArrayType()) { + if (!(literal instanceof ArrayLiteral)) { + throw new AnalysisException(literal.toSql() + " is not an array literal"); + } + DataType itemType = ((ArrayType) type).getItemType(); + StringJoiner joiner = new StringJoiner(", ", "[", "]"); + for (Literal item : ((ArrayLiteral) literal).getValue()) { + joiner.add(render(item, itemType)); + } + return joiner.toString(); + } + if (type.isMapType()) { + if (!(literal instanceof MapLiteral)) { + throw new AnalysisException(literal.toSql() + " is not a map literal"); + } + MapType mapType = (MapType) type; + // The parser keeps the last value of a repeated key, so a repeated key is stored once. Keys that + // only collide after the cast to the key type (e.g. "01" and "1" for INT) are rejected: BE keeps + // every entry of the stored text, while replaying it as a SQL literal would keep only the last. + Set keys = new HashSet<>(); + StringJoiner joiner = new StringJoiner(", ", "{", "}"); + for (Map.Entry entry : ((MapLiteral) literal).getValue().entrySet()) { + String key = render(entry.getKey(), mapType.getKeyType()); + if (!keys.add(key)) { + throw new AnalysisException(String.format("map key %s is repeated after casting to %s", key, + mapType.getKeyType().toSql())); + } + joiner.add(key + ":" + render(entry.getValue(), mapType.getValueType())); + } + return joiner.toString(); + } + if (type.isStructType()) { + // `{}` parses as an empty map literal and means every field takes its own default. + if (literal instanceof MapLiteral && ((MapLiteral) literal).getValue().isEmpty()) { + return "{}"; + } + if (!(literal instanceof StructLiteral)) { + throw new AnalysisException(literal.toSql() + " is not a struct literal"); + } + List fields = ((StructType) type).getFields(); + List values = ((StructLiteral) literal).getValue(); + if (values.size() != fields.size()) { + throw new AnalysisException(String.format("struct literal has %d fields but the column has %d", + values.size(), fields.size())); + } + StringJoiner joiner = new StringJoiner(", ", "{", "}"); + for (int i = 0; i < fields.size(); i++) { + joiner.add(render(values.get(i), fields.get(i).getDataType())); + } + return joiner.toString(); + } + return renderScalar(literal, type); + } + + private static String renderScalar(Literal literal, DataType type) throws AnalysisException { + if (type.isStringLikeType()) { + // The cast to VARCHAR(n) silently truncates a longer value, while a scalar VARCHAR(n) column + // rejects such a default, so cast to STRING and check the length against the declared type. + String value = ((Literal) literal.checkedCastTo(StringType.INSTANCE)).getStringValue(); + int maxLength = ((CharacterType) type).getLen(); + if (maxLength >= 0 && value.length() > maxLength) { + throw new AnalysisException("Default value is too long: " + value); + } + return renderQuoted(literal, value); + } + literal = (Literal) literal.checkedCastTo(type); + if (type.isBooleanType()) { + return ((BooleanLiteral) literal).getValue() ? "1" : "0"; + } + if (type.isNumericType()) { + if (type.isFloatLikeType()) { + double value = ((Number) literal.getValue()).doubleValue(); + if (!Double.isFinite(value)) { + throw new AnalysisException(literal.toSql() + " is not a finite number"); + } + // A SQL numeric literal has no negative zero: replaying the "-0.0" of SHOW CREATE TABLE + // output parses it through BigDecimal and stores 0.0, so render -0.0 as 0.0 from the start. + if (value == 0) { + literal = type.isFloatType() ? new FloatLiteral(0.0f) : new DoubleLiteral(0.0); + } + } + return literal.getStringValue(); + } + if (type.isDateType() || type.isDateV2Type() || type.isDateTimeType() || type.isDateTimeV2Type() + || type.isIPType()) { + return renderQuoted(literal, literal.getStringValue()); + } + throw new AnalysisException(type.toSql() + " is not supported as a nested type of a default value"); + } + + private static String renderQuoted(Literal literal, String value) throws AnalysisException { + if (value.indexOf('"') >= 0 || value.indexOf('\'') >= 0 || value.indexOf('\\') >= 0) { + throw new AnalysisException("string value " + literal.toSql() + + " must not contain quote or backslash"); + } + return "\"" + value + "\""; + } + + /** + * Builds array and map literals from their items as written. + * + *

{@link LogicalPlanBuilder} first casts the items of an array literal, and the keys and the values of a + * map literal, to a common type inferred from the items, and with {@code enable_strict_cast=false} an item + * that cannot be cast silently becomes NULL: {@code '[DATEV2 "2024-01-01", 1]'} would be stored as + * {@code ["2024-01-01", NULL]}. The items of a default value are cast to the declared nested type by + * {@link #render} instead, which rejects such an item, so no coercion happens while parsing. + */ + private static final class LiteralBuilder extends LogicalPlanBuilder { + private LiteralBuilder() { + super(ImmutableMap.of()); + } + + @Override + public ArrayLiteral visitArrayLiteral(ArrayLiteralContext ctx) { + return new ArrayLiteral(visitItems(ctx, ctx.items)); + } + + @Override + public MapLiteral visitMapLiteral(MapLiteralContext ctx) { + // the grammar only accepts complete key:value pairs + List items = visitItems(ctx, ctx.items); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < items.size(); i += 2) { + map.put(items.get(i), items.get(i + 1)); + } + return new MapLiteral(map); + } + + private List visitItems(ParserRuleContext ctx, List items) { + ImmutableList.Builder literals = ImmutableList.builderWithExpectedSize(items.size()); + for (ConstantContext item : items) { + Expression expression = typedVisit(item); + // a placeholder is the only constant that is not a literal + if (!(expression instanceof Literal)) { + throw new ParseException("default value must only contain literals", ctx); + } + literals.add((Literal) expression); + } + return literals.build(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java index d78c92d0213bab..0f1fac2d08aafb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java @@ -224,6 +224,17 @@ public void testStrictNestedPrimitivePromotionRules() { ScalarType.createDecimalV3Type(5, 2), ScalarType.createDecimalV3Type(10, 3))); } + @Test + public void testToSqlQuotesDefaultValueForReplay() { + Column plain = new Column("c1", Type.STRING, false, null, true, "abc", ""); + Assertions.assertTrue(plain.toSql().contains(" DEFAULT \"abc\""), plain.toSql()); + Column complex = new Column("c2", new MapType(Type.STRING, Type.INT), false, null, true, + "{\"a\":1}", ""); + Assertions.assertTrue(complex.toSql().contains(" DEFAULT '{\"a\":1}'"), complex.toSql()); + Column quoted = new Column("c3", Type.STRING, false, null, true, "it's", ""); + Assertions.assertTrue(quoted.toSql().contains(" DEFAULT \"it's\""), quoted.toSql()); + } + @Test public void testIcebergNestedDecimalPromotionRules() { Assertions.assertTrue(ColumnType.isSupportedIcebergNestedDecimalPromotion( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java index 8a785370a9fff3..a2c8c5d06d7246 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java @@ -18,16 +18,22 @@ package org.apache.doris.nereids.load; import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.fileformat.ArrowFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; @@ -152,6 +158,63 @@ public void testImportColumnMatchesTableColumnCaseInsensitive() { ImmutableList.of(new NereidsImportColumnDesc("other")), tableColumn)); } + @Test + public void testComplexColumnMappingValueIsCanonicalized() throws Exception { + OlapTable table = mockComplexTable(); + StringLiteral canonical = new StringLiteral("[1000, 7]"); + + // v = default_value('[1e3, "7"]') + NereidsParamCreateContext context = createMappingLoadContext(table, + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("[1e3, \"7\"]"))), + Pair.of("default_value", Lists.newArrayList("[1e3, \"7\"]"))); + Assertions.assertEquals(canonical, context.exprMap.get("v")); + + // v = replace_value(null, '[1e3, "7"]') becomes if(v is not null, v, '[1000, 7]') + context = createMappingLoadContext(table, + new UnboundFunction("replace_value", ImmutableList.of(new NullLiteral(), + new StringLiteral("[1e3, \"7\"]"))), + Pair.of("replace_value", Lists.newArrayList(null, "[1e3, \"7\"]"))); + Expression replaced = context.exprMap.get("v"); + Assertions.assertTrue(replaced instanceof UnboundFunction, replaced.toSql()); + Assertions.assertEquals(canonical, replaced.child(2)); + + // v = replace_value('[]', '[1e3, "7"]') becomes if(v is not null, if(v != '[]', v, '[1000, 7]'), null) + context = createMappingLoadContext(table, + new UnboundFunction("replace_value", ImmutableList.of(new StringLiteral("[]"), + new StringLiteral("[1e3, \"7\"]"))), + Pair.of("replace_value", Lists.newArrayList("[]", "[1e3, \"7\"]"))); + replaced = context.exprMap.get("v"); + Assertions.assertTrue(replaced instanceof UnboundFunction, replaced.toSql()); + Assertions.assertEquals(canonical, replaced.child(1).child(2)); + + // the value is validated as a literal of the column type before it is rewritten + UserException exception = Assertions.assertThrows(UserException.class, () -> createMappingLoadContext(table, + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("[\"bad\"]"))), + Pair.of("default_value", Lists.newArrayList("[\"bad\"]")))); + Assertions.assertTrue(exception.getMessage().contains("Invalid default value"), exception.getMessage()); + + // a scalar column keeps its mapping value verbatim + context = createMappingLoadContext(table, "k", + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("007"))), + Pair.of("default_value", Lists.newArrayList("007"))); + Assertions.assertEquals(new StringLiteral("007"), context.exprMap.get("k")); + } + + private NereidsParamCreateContext createMappingLoadContext(OlapTable table, Expression mapping, + Pair> hadoopFunction) throws UserException { + return createMappingLoadContext(table, "v", mapping, hadoopFunction); + } + + private NereidsParamCreateContext createMappingLoadContext(OlapTable table, String column, Expression mapping, + Pair> hadoopFunction) throws UserException { + Map>> columnToHadoopFunction = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + columnToHadoopFunction.put(column, hadoopFunction); + return createLoadContext(table, + ImmutableList.of(new NereidsImportColumnDesc("k"), new NereidsImportColumnDesc("v"), + new NereidsImportColumnDesc(column, mapping)), + columnToHadoopFunction, new ArrowFileFormatProperties()); + } + private NereidsParamCreateContext createArrowLoadContext( OlapTable table, List columnExprList) throws UserException { return createLoadContext(table, columnExprList, new ArrowFileFormatProperties()); @@ -160,9 +223,15 @@ private NereidsParamCreateContext createArrowLoadContext( private NereidsParamCreateContext createLoadContext(OlapTable table, List columnExprList, FileFormatProperties fileFormatProperties) throws UserException { + return createLoadContext(table, columnExprList, null, fileFormatProperties); + } + + private NereidsParamCreateContext createLoadContext(OlapTable table, List columnExprList, + Map>> columnToHadoopFunction, FileFormatProperties fileFormatProperties) + throws UserException { NereidsBrokerFileGroup fileGroup = new NereidsBrokerFileGroup(1L, false, null, - Lists.newArrayList("dummy"), null, null, null, columnExprList, null, null, null, null, - LoadTask.MergeType.APPEND, null, -1L, false, false, fileFormatProperties); + Lists.newArrayList("dummy"), null, null, null, columnExprList, columnToHadoopFunction, null, null, + null, LoadTask.MergeType.APPEND, null, -1L, false, false, fileFormatProperties); TBrokerFileStatus fileStatus = new TBrokerFileStatus(); fileStatus.setPath(""); fileStatus.setIsDir(false); @@ -174,10 +243,13 @@ private NereidsParamCreateContext createLoadContext(OlapTable table, List schema = Arrays.asList( + return mockTable(Arrays.asList( new Column("time", PrimitiveType.DATETIME, true), new Column("securityid", PrimitiveType.INT, true), - new Column("EV", PrimitiveType.DOUBLE, true)); + new Column("EV", PrimitiveType.DOUBLE, true))); + } + + private OlapTable mockTable(List schema) { Map nameToColumn = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (Column column : schema) { nameToColumn.put(column.getName(), column); @@ -198,6 +270,12 @@ private OlapTable mockTable() { return table; } + private OlapTable mockComplexTable() { + return mockTable(Arrays.asList( + new Column("k", PrimitiveType.INT, true), + new Column("v", ArrayType.create(ScalarType.INT, true), true))); + } + private void assertSlot(NereidsParamCreateContext context, String name, PrimitiveType type) { SlotReference slot = context.scanSlots.stream() .filter(scanSlot -> scanSlot.getName().equals(name)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index ace25f884b30cc..f9bf211e5aada4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -316,7 +316,7 @@ void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisExcepti () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( "ALTER TABLE t MODIFY COLUMN st STRUCT DEFAULT 'x'").getOps())); Assertions.assertTrue(complexDefaultException.getMessage() - .contains("Struct type column default value just support null")); + .contains("Struct type column default value only supports struct literals or DEFAULT NULL")); AnalysisException onUpdateException = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index 5a919c81a6d1e2..dd1b0f6d0d5194 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -22,12 +22,26 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.types.AggStateType; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.HllType; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.JsonType; +import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.QuantileStateType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.UuidType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -36,6 +50,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Arrays; +import java.util.Collections; import java.util.Optional; public class ColumnDefinitionTest { @@ -152,4 +168,124 @@ public void testAddColumnRejectsUuidDynamicDefaults() { Optional.of(new DefaultValue("00112233-4455-6677-8899-aabbccddeeff")), ""); Assertions.assertFalse(literal.hasUuidDefaultValue()); } + + @Test + public void testComplexTypeLiteralDefaultValue() { + ArrayType intArray = ArrayType.of(IntegerType.INSTANCE); + assertCanonicalDefaultValue(intArray, "[]", "[]"); + assertCanonicalDefaultValue(intArray, "[1, 2]", "[1, 2]"); + // nested NULL is rendered in lower case: the BE JSON parser that fills the omitted columns of a + // partial update only recognizes `null`, and `NULL` would be stored as the string "NULL" + assertCanonicalDefaultValue(intArray, "[NULL, nUlL, 5]", "[null, null, 5]"); + assertCanonicalDefaultValue(intArray, "[\"7\", 1e3]", "[7, 1000]"); + assertRejectsDefaultValue(intArray, "{}", "only supports array literals or DEFAULT NULL"); + assertRejectsDefaultValue(intArray, "[1 + 1]", "only supports array literals or DEFAULT NULL"); + assertRejectsDefaultValue(intArray, "[\"bad\"]", "Invalid default value '[\"bad\"]' for ARRAY"); + assertRejectsDefaultValue(intArray, "[[1]]", "Invalid default value"); + + assertCanonicalDefaultValue(ArrayType.of(DateV2Type.INSTANCE), + "[DATEV2 \"2024-01-01\", \"2024-02-02\"]", "[\"2024-01-01\", \"2024-02-02\"]"); + // an item that cannot be cast to the type inferred from the other items is rejected instead of + // being replaced by NULL, which the SQL literal parser does when enable_strict_cast is off + Assertions.assertFalse(SessionVariable.enableStrictCast()); + assertRejectsDefaultValue(ArrayType.of(DateV2Type.INSTANCE), "[DATEV2 \"2024-01-01\", 1]", + "Invalid default value '[DATEV2 \"2024-01-01\", 1]' for ARRAY column"); + // the parser would infer DATEV2 for the items and turn 1 into NULL; each item is cast to INT instead + assertCanonicalDefaultValue(intArray, "[DATEV2 \"2024-01-01\", 1]", "[20240101, 1]"); + assertRejectsDefaultValue(ArrayType.of(ArrayType.of(DateV2Type.INSTANCE)), + "[[DATEV2 \"2024-01-01\"], [1]]", "Invalid default value"); + assertCanonicalDefaultValue(ArrayType.of(BooleanType.INSTANCE), "[true, false]", "[1, 0]"); + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[1.5, 2]", "[1.5, 2.0]"); + // a SQL numeric literal has no negative zero, so the canonical text never contains -0.0 + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[\"-0\", -0.0, \"-0.0\"]", + "[0.0, 0.0, 0.0]"); + assertCanonicalDefaultValue(ArrayType.of(FloatType.INSTANCE), "[\"-0\", -0.0]", "[0.0, 0.0]"); + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[\"-1.5\"]", "[-1.5]"); + assertCanonicalDefaultValue(ArrayType.of(DecimalV3Type.createDecimalV3Type(10, 2)), "[1.234]", "[1.23]"); + assertCanonicalDefaultValue(ArrayType.of(StringType.INSTANCE), "['x,y', \"[z]\", \"{k:v}\", '']", + "[\"x,y\", \"[z]\", \"{k:v}\", \"\"]"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "[\"a\"\"b\"]", + "must not contain quote or backslash"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "['it''s']", + "must not contain quote or backslash"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "['a\\\\b']", + "must not contain quote or backslash"); + assertCanonicalDefaultValue(ArrayType.of(intArray), "[[1], [], NULL]", "[[1], [], null]"); + // a nested VARCHAR(n)/CHAR(n) value longer than n is rejected like a scalar default, instead of + // being truncated by the cast to VARCHAR(n) + assertCanonicalDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[\"ab\", 1]", + "[\"ab\", \"1\"]"); + assertRejectsDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[\"abc\"]", + "Default value is too long: abc"); + assertRejectsDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[123]", + "Default value is too long: 123"); + assertCanonicalDefaultValue(ArrayType.of(CharType.createCharType(2)), "[\"ab\"]", "[\"ab\"]"); + assertRejectsDefaultValue(ArrayType.of(CharType.createCharType(2)), "[\"abc\"]", + "Default value is too long: abc"); + + MapType stringIntMap = MapType.of(StringType.INSTANCE, IntegerType.INSTANCE); + assertCanonicalDefaultValue(stringIntMap, "{}", "{}"); + assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"b\": NULL}", "{\"a\":1, \"b\":null}"); + assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"a\": 2}", "{\"a\":2}"); + assertRejectsDefaultValue(stringIntMap, "[]", "only supports map literals or DEFAULT NULL"); + assertRejectsDefaultValue(stringIntMap, "{\"a\": \"bad\"}", "Invalid default value"); + assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"bad\": 1}", + "Invalid default value"); + assertRejectsDefaultValue(MapType.of(DateV2Type.INSTANCE, IntegerType.INSTANCE), + "{DATEV2 \"2024-01-01\": 1, 1: 2}", "Invalid default value"); + assertRejectsDefaultValue(MapType.of(StringType.INSTANCE, DateV2Type.INSTANCE), + "{\"a\": DATEV2 \"2024-01-01\", \"b\": 1}", "Invalid default value"); + // keys that are distinct literals but collide after the cast to the key type are rejected + assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"01\": 1, \"1\": 2}", + "map key 1 is repeated after casting to INT"); + assertCanonicalDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"01\": 1, 2: 2}", + "{1:1, 2:2}"); + + StructType structType = new StructType(Arrays.asList( + new StructField("f1", IntegerType.INSTANCE, true, ""), + new StructField("f2", StringType.INSTANCE, true, ""))); + assertCanonicalDefaultValue(structType, "{}", "{}"); + assertCanonicalDefaultValue(structType, "{1, \"a\"}", "{1, \"a\"}"); + assertCanonicalDefaultValue(structType, "{NULL, 2}", "{null, \"2\"}"); + assertRejectsDefaultValue(structType, "{\"f1\": 1, \"f2\": \"a\"}", + "only supports struct literals or DEFAULT NULL"); + assertRejectsDefaultValue(structType, "{1}", "struct literal has 1 fields but the column has 2"); + assertRejectsDefaultValue(structType, "{\"bad\", \"a\"}", "Invalid default value"); + assertRejectsDefaultValue(new StructType(Arrays.asList( + new StructField("f1", ArrayType.of(DateV2Type.INSTANCE), true, ""))), + "{[DATEV2 \"2024-01-01\", 1]}", "Invalid default value"); + assertRejectsDefaultValue(structType, "[]", "only supports struct literals or DEFAULT NULL"); + + assertRejectsDefaultValue(JsonType.INSTANCE, "{}", "only supports DEFAULT NULL"); + assertRejectsDefaultValue(VariantType.INSTANCE, "{}", "only supports DEFAULT NULL"); + + Assertions.assertDoesNotThrow(() -> newColumnDefinition( + ArrayType.of(IntegerType.INSTANCE), Optional.empty()).validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + Assertions.assertDoesNotThrow(() -> newColumnDefinition( + MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), + Optional.of(DefaultValue.NULL_DEFAULT_VALUE)).validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + } + + private void assertCanonicalDefaultValue(DataType type, String defaultValue, String canonical) { + ColumnDefinition columnDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(defaultValue))); + columnDefinition.validate(true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS); + Assertions.assertEquals(canonical, columnDefinition.getDefaultValueString()); + // the canonical text is itself a valid default and is stable + ColumnDefinition canonicalDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(canonical))); + canonicalDefinition.validate(true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS); + Assertions.assertEquals(canonical, canonicalDefinition.getDefaultValueString()); + } + + private void assertRejectsDefaultValue(DataType type, String defaultValue, String message) { + ColumnDefinition columnDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(defaultValue))); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> columnDefinition.validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + Assertions.assertTrue(exception.getMessage().contains(message), exception.getMessage()); + } + + private ColumnDefinition newColumnDefinition(DataType type, Optional defaultValue) { + return new ColumnDefinition("col1", type, false, null, true, defaultValue, ""); + } } diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out new file mode 100644 index 00000000000000..71c9190bc7f824 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out @@ -0,0 +1,42 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !literal_default -- +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] ["ab", "1"] [0, 0, -1.5] + +-- !like_default -- +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] ["ab", "1"] [0, 0, -1.5] + +-- !null_default -- +1 \N \N \N \N \N + +-- !alter_default -- +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} + +-- !alter_default_after_insert -- +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} +2 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} + +-- !alter_default_pruned -- +1 y \N ["p", "q"] r 2 false +2 y \N ["p", "q"] r 2 false + +-- !alter_direct_default -- +1 [3, 4] {"a":2} {"f1":8, "f2":["2024-01-01"]} ["x,y", ""] + +-- !replace_value -- +1 {"a":1} +2 {"b":2} + +-- !mapping_replace_value -- +1 [1000, 7] +2 [5] + +-- !mapping_default_value -- +1 [1000, 7] +1 [1000, 7] +2 [1000, 7] +2 [5] + +-- !partial_update_default -- +1 [null, "null", "x"] true {"a":null, "b":"y"} {"f1":null, "f2":null} true {"f1":7, "f2":"x"} [1, 2] +2 [null, "null", "x"] true {"a":null, "b":"y"} {"f1":null, "f2":null} true {"f1":7, "f2":"x"} [1, 2] + diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv new file mode 100644 index 00000000000000..2e4e963e54f98a --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv @@ -0,0 +1,2 @@ +1 \N +2 [5] diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv new file mode 100644 index 00000000000000..4d41303fa8fd31 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv @@ -0,0 +1,2 @@ +1 \N +2 {"b":2} diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy new file mode 100644 index 00000000000000..477ed86dbb2707 --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -0,0 +1,366 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_complex_default_value") { + sql "DROP TABLE IF EXISTS test_complex_default_value_literal" + sql "DROP TABLE IF EXISTS test_complex_default_value_like" + sql "DROP TABLE IF EXISTS test_complex_default_value_null" + sql "DROP TABLE IF EXISTS test_complex_default_value_alter_base" + sql "DROP TABLE IF EXISTS test_complex_default_value_alter_direct" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter" + sql "DROP TABLE IF EXISTS test_complex_default_value_replace_value" + sql "DROP TABLE IF EXISTS test_complex_default_value_mapping" + sql "DROP TABLE IF EXISTS test_complex_default_value_partial_update" + + def createTableRejects = { String columnDef, String message -> + test { + sql """ + CREATE TABLE test_complex_default_value_rejected ( + k INT, + v ${columnDef} + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception message + } + } + + // the default must be a literal of the column's own shape + createTableRejects.call("ARRAY DEFAULT '{}'", "only supports array literals or DEFAULT NULL") + createTableRejects.call("ARRAY DEFAULT '[1 + 1]'", "only supports array literals or DEFAULT NULL") + createTableRejects.call("MAP DEFAULT '[]'", "only supports map literals or DEFAULT NULL") + createTableRejects.call("STRUCT DEFAULT '[]'", "only supports struct literals or DEFAULT NULL") + createTableRejects.call("""STRUCT DEFAULT '{"f1": 1, "f2": "a"}'""", "only supports struct literals or DEFAULT NULL") + createTableRejects.call("JSON DEFAULT '{}'", "only supports DEFAULT NULL") + createTableRejects.call("VARIANT DEFAULT '{}'", "only supports DEFAULT NULL") + + // every nested value is cast to the declared nested type at DDL time + createTableRejects.call("""ARRAY DEFAULT '["bad"]'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"bad": 1}'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"bad": "value"}'""", "Invalid default value") + createTableRejects.call("""STRUCT DEFAULT '{"bad"}'""", "Invalid default value") + createTableRejects.call("""STRUCT DEFAULT '{1}'""", "struct literal has 1 fields but the column has 2") + createTableRejects.call("""ARRAY> DEFAULT '[[1], ["bad"]]'""", "Invalid default value") + // an item that cannot be cast to the type inferred from the other items is rejected instead of being + // replaced by NULL, which the SQL literal parser does under the default enable_strict_cast=false + createTableRejects.call("""ARRAY DEFAULT '[DATEV2 "2024-01-01", 1]'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{DATEV2 "2024-01-01": 1, 1: 2}'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"a": DATEV2 "2024-01-01", "b": 1}'""", "Invalid default value") + createTableRejects.call("""STRUCT> DEFAULT '{[DATEV2 "2024-01-01", 1]}'""", "Invalid default value") + // map keys that only collide after the cast to the key type are rejected + createTableRejects.call("""MAP DEFAULT '{"01": 1, "1": 2}'""", "map key 1 is repeated after casting to INT") + // a nested value longer than the declared VARCHAR(n)/CHAR(n) is rejected like a scalar default instead of + // being truncated + createTableRejects.call("""ARRAY DEFAULT '["abc"]'""", "Default value is too long: abc") + createTableRejects.call("""ARRAY DEFAULT '["abc"]'""", "Default value is too long: abc") + createTableRejects.call("""STRUCT DEFAULT '{123}'""", "Default value is too long: 123") + + // nested string values are stored as plain double quoted text, so quotes and backslashes are rejected + createTableRejects.call("""ARRAY DEFAULT '["a""b"]'""", "must not contain quote or backslash") + createTableRejects.call("""MAP DEFAULT '{"a\\\\\\\\b": 1}'""", "must not contain quote or backslash") + createTableRejects.call("""ARRAY DEFAULT '["it''s"]'""", "must not contain quote or backslash") + + // ADD COLUMN validates the default the same way, for both light and direct schema change + sql """ + CREATE TABLE test_complex_default_value_bad_alter ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1', 'light_schema_change'='false') + """ + sql "INSERT INTO test_complex_default_value_bad_alter VALUES (1)" + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '["bad"]'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"bad": 1}'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v STRUCT DEFAULT '{"bad"}'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"01": 1, "1": 2}'""" + exception "map key 1 is repeated after casting to INT" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '[DATEV2 "2024-01-01", 1]'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"a": DATEV2 "2024-01-01", "b": 1}'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '["abc"]'""" + exception "Default value is too long: abc" + } + + sql """ + CREATE TABLE test_complex_default_value_literal ( + k INT, + arr_empty ARRAY DEFAULT '[]', + arr_literal ARRAY DEFAULT '[1, 2]', + map_empty MAP DEFAULT '{}', + map_literal MAP DEFAULT '{"a": 10, "b": 20}', + struct_empty STRUCT DEFAULT '{}', + struct_literal STRUCT DEFAULT '{7, "x"}', + arr_nested_null ARRAY DEFAULT '[NULL, nUlL, 5]', + map_nested_null MAP DEFAULT '{"upper": NULL, "mixed": nUlL, "value": 6}', + struct_nested_null STRUCT DEFAULT '{NULL, NuLl, 7}', + arr_typed_date ARRAY DEFAULT '[DATEV2 "2024-01-01", "2024-02-02"]', + arr_exponent ARRAY DEFAULT '[1e3, "7"]', + arr_bool ARRAY DEFAULT '[true, false]', + arr_decimal ARRAY DEFAULT '[1.234]', + arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', + map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + arr_nested ARRAY> DEFAULT '[[1], [], NULL]', + arr_varchar ARRAY DEFAULT '["ab", 1]', + arr_negative_zero ARRAY DEFAULT '["-0", -0.0, "-1.5"]' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + + sql "INSERT INTO test_complex_default_value_literal(k) VALUES (1)" + order_qt_literal_default """ + SELECT * FROM test_complex_default_value_literal ORDER BY k + """ + + // SHOW CREATE TABLE renders the canonical default in a form that CREATE TABLE LIKE can replay + def showCreate = sql "SHOW CREATE TABLE test_complex_default_value_literal" + def createSql = showCreate[0][1] + logger.info("show create table: ${createSql}") + assertTrue(createSql.contains('`arr_literal` array NULL DEFAULT "[1, 2]"')) + assertTrue(createSql.contains('`map_literal` map NULL DEFAULT \'{"a":10, "b":20}\'')) + assertTrue(createSql.contains('`arr_typed_date` array NULL DEFAULT \'["2024-01-01", "2024-02-02"]\'')) + assertTrue(createSql.contains('`arr_exponent` array NULL DEFAULT "[1000, 7]"')) + assertTrue(createSql.contains('`arr_bool` array NULL DEFAULT "[1, 0]"')) + assertTrue(createSql.contains('`map_repeated_key` map NULL DEFAULT \'{"a":2}\'')) + assertTrue(createSql.contains('`arr_string` array NULL DEFAULT \'["x,y", "[z]", "{k:v}", "null", ""]\'')) + // nested NULL is stored in lower case, the only spelling every BE consumer of the text accepts + assertTrue(createSql.contains('`arr_nested_null` array NULL DEFAULT "[null, null, 5]"')) + assertTrue(createSql.contains('`struct_nested_null` struct NULL DEFAULT "{null, null, 7}"')) + // a SQL numeric literal has no negative zero, so -0.0 is stored as 0.0 and replays unchanged + assertTrue(createSql.contains('`arr_negative_zero` array NULL DEFAULT "[0.0, 0.0, -1.5]"')) + sql "CREATE TABLE test_complex_default_value_like LIKE test_complex_default_value_literal" + sql "INSERT INTO test_complex_default_value_like(k) VALUES (1)" + order_qt_like_default """ + SELECT * FROM test_complex_default_value_like ORDER BY k + """ + + sql """ + CREATE TABLE test_complex_default_value_null ( + k INT, + arr_col ARRAY DEFAULT NULL, + map_col MAP DEFAULT NULL, + struct_col STRUCT DEFAULT NULL, + json_col JSON DEFAULT NULL, + variant_col VARIANT DEFAULT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + + sql "INSERT INTO test_complex_default_value_null(k) VALUES (1)" + order_qt_null_default """ + SELECT k, arr_col, map_col, struct_col, json_col, variant_col + FROM test_complex_default_value_null + ORDER BY k + """ + + // rows written before the columns were added read the defaults through the BE default value iterator + sql """ + CREATE TABLE test_complex_default_value_alter_base ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='true' + ) + """ + + sql "INSERT INTO test_complex_default_value_alter_base VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_alter_base + ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', + ADD COLUMN map_added MAP NOT NULL DEFAULT '{"z": 9}', + ADD COLUMN struct_added STRUCT NOT NULL DEFAULT '{8, "y"}', + ADD COLUMN arr_typed_date ARRAY DEFAULT '[DATEV2 "2024-01-01", "2024-02-02"]', + ADD COLUMN arr_exponent ARRAY DEFAULT '[1e3, "7"]', + ADD COLUMN arr_bool ARRAY DEFAULT '[true, false]', + ADD COLUMN arr_decimal ARRAY DEFAULT '[1.234]', + ADD COLUMN arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', + ADD COLUMN map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + ADD COLUMN arr_nested ARRAY> DEFAULT '[[1], [], NULL]', + ADD COLUMN struct_empty STRUCT DEFAULT '{}', + ADD COLUMN arr_struct ARRAY> DEFAULT '[{1, "p"}, {2, "q"}]', + ADD COLUMN map_struct MAP> DEFAULT '{"k": {3, "r"}}' + """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_base' ORDER BY createtime DESC LIMIT 1""" + time 600 + } + + order_qt_alter_default """ + SELECT * FROM test_complex_default_value_alter_base ORDER BY k + """ + + sql "INSERT INTO test_complex_default_value_alter_base(k) VALUES (2)" + order_qt_alter_default_after_insert """ + SELECT * FROM test_complex_default_value_alter_base ORDER BY k + """ + // nested column pruning reads only the accessed STRUCT fields, so the default of an old row is + // projected onto the pruned STRUCT, also inside ARRAY items and MAP values + order_qt_alter_default_pruned """ + SELECT k, + struct_element(struct_added, 'f2'), + struct_element(struct_empty, 'f2'), + array_map(x -> struct_element(x, 'f2'), arr_struct), + struct_element(element_at(map_struct, 'k'), 'f2'), + size(arr_struct), + struct_added IS NULL + FROM test_complex_default_value_alter_base ORDER BY k + """ + + // direct schema change materializes the defaults into the rewritten rows + sql """ + CREATE TABLE test_complex_default_value_alter_direct ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='false' + ) + """ + + sql "INSERT INTO test_complex_default_value_alter_direct VALUES (1)" + // a delete predicate on the base rowsets makes BE pick the direct strategy instead of linking + // the old rowset, so the new columns are materialized by the schema change itself + sql "DELETE FROM test_complex_default_value_alter_direct WHERE k = 0" + sql """ + ALTER TABLE test_complex_default_value_alter_direct + ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', + ADD COLUMN map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + ADD COLUMN struct_added STRUCT> DEFAULT '{8, [DATEV2 "2024-01-01"]}', + ADD COLUMN arr_string ARRAY DEFAULT '["x,y", ""]' + """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_direct' ORDER BY createtime DESC LIMIT 1""" + time 600 + } + + order_qt_alter_direct_default """ + SELECT * FROM test_complex_default_value_alter_direct ORDER BY k + """ + + // a one-argument replace_value mapping falls back to the complex column default + sql """ + CREATE TABLE test_complex_default_value_replace_value ( + k INT, + v MAP DEFAULT '{"a": 1}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + streamLoad { + table "test_complex_default_value_replace_value" + set 'columns', 'k, v, v = replace_value(null)' + file "test_complex_default_value_replace_value.csv" + time 60 + } + order_qt_replace_value """ + SELECT k, v FROM test_complex_default_value_replace_value ORDER BY k + """ + + // an explicit default_value / replace_value argument of a complex column is written as the + // canonical literal, so 1e3 and "7" become INT elements instead of NULL + sql """ + CREATE TABLE test_complex_default_value_mapping ( + k INT, + v ARRAY + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + streamLoad { + table "test_complex_default_value_mapping" + set 'columns', 'k, v, v = replace_value(null, \'[1e3, "7"]\')' + file "test_complex_default_value_mapping.csv" + time 60 + } + order_qt_mapping_replace_value """ + SELECT k, v FROM test_complex_default_value_mapping ORDER BY k + """ + streamLoad { + table "test_complex_default_value_mapping" + set 'columns', 'k, tmp, v = default_value(\'[1e3, "7"]\')' + file "test_complex_default_value_mapping.csv" + time 60 + } + order_qt_mapping_default_value """ + SELECT k, v FROM test_complex_default_value_mapping ORDER BY k, v + """ + test { + sql """ + ALTER TABLE test_complex_default_value_mapping + ADD COLUMN bad MAP DEFAULT '{"01": 1, "1": 2}' + """ + exception "map key 1 is repeated after casting to INT" + } + + // a partial update of a new key fills the omitted complex columns from their defaults through the + // BE JSON default parser, which must read a nested null the same way as the default value iterator + sql """ + CREATE TABLE test_complex_default_value_partial_update ( + k INT, + v INT, + arr_nested_null ARRAY DEFAULT '[NULL, "null", "x"]', + map_nested_null MAP DEFAULT '{"a": NULL, "b": "y"}', + struct_nested_null STRUCT DEFAULT '{NULL, NULL}', + struct_literal STRUCT DEFAULT '{7, "x"}', + arr_literal ARRAY DEFAULT '[1, 2]' + ) + UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1', 'enable_unique_key_merge_on_write'='true') + """ + sql "INSERT INTO test_complex_default_value_partial_update(k, v) VALUES (1, 1)" + sql "SET enable_unique_key_partial_update = true" + sql "SET enable_insert_strict = false" + sql "INSERT INTO test_complex_default_value_partial_update(k) VALUES (2)" + sql "SET enable_unique_key_partial_update = false" + sql "SET enable_insert_strict = true" + order_qt_partial_update_default """ + SELECT k, arr_nested_null, array_contains(arr_nested_null, NULL), map_nested_null, + struct_nested_null, struct_element(struct_nested_null, 'f2') IS NULL, struct_literal, arr_literal + FROM test_complex_default_value_partial_update ORDER BY k + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy index b79f9f1135de84..9045b4a7e2fc33 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy @@ -77,7 +77,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do STRUCT(1, CAST(1.1 AS FLOAT), 'abc', MAP(1, 10), ARRAY(1, 2)) )""" - // Complex type default value only supports NULL + // Invalid complex type default value is rejected. test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT< @@ -87,7 +87,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do sm: MAP COMMENT 'm1', sa: ARRAY COMMENT 'a1' > DEFAULT 'x'""" - exception "just support null" + exception "only supports struct literals or DEFAULT NULL" } // Cannot change nullable complex column to not null diff --git a/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy b/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy index bcc5583a6fc56b..b2bb2ed20e633b 100644 --- a/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy @@ -587,7 +587,7 @@ suite("test_dup_schema_key_add","p0") { //TODO Test the duplicate model by adding a column with JSON type none default value - errorMessage="errCode = 2, detailMessage = Json or Variant type column default value just support null" + errorMessage="errCode = 2, detailMessage = Json or Variant type column default value only supports DEFAULT NULL" expectException({ sql initTable sql initTableData diff --git a/regression-test/suites/schema_change_p0/test_modify_struct.groovy b/regression-test/suites/schema_change_p0/test_modify_struct.groovy index 12effd00a15cc9..0a750b2bc21df4 100644 --- a/regression-test/suites/schema_change_p0/test_modify_struct.groovy +++ b/regression-test/suites/schema_change_p0/test_modify_struct.groovy @@ -306,10 +306,10 @@ suite ("test_modify_struct") { String[][] descRes = sql """ desc ${table_name} """ logger.info(descRes[2][2]) assertEquals(descRes[2][2].toString().toLowerCase(), "no") - // sc do not support new struct type with not null. - def changeNotNullErr = "Struct type column default value just support null" + // Invalid struct default value is rejected. + def changeNotNullErr = "Struct type column default value only supports struct literals or DEFAULT NULL" expectExceptionLike({ - sql """ alter table ${table_name} add column c_s_not_null1 struct NOT NULL default '{}'""" + sql """ alter table ${table_name} add column c_s_not_null1 struct NOT NULL default 'x'""" waitUntilSchemaChangeDone.call(table_name, "") }, changeNotNullErr) diff --git a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy index 7a39ab2b11dd2b..b462e0a4674b60 100644 --- a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy +++ b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy @@ -256,7 +256,7 @@ suite("test_unique_schema_key_change_add","p0") { //TODO Test the unique model by adding a column with JSON type none default value - errorMessage="errCode = 2, detailMessage = Json or Variant type column default value just support null" + errorMessage="errCode = 2, detailMessage = Json or Variant type column default value only supports DEFAULT NULL" expectException({ sql initTable sql initTableData