From a0b79ec1d2c5416f9cb121ca64faf2c95e98ed2b Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Wed, 27 May 2026 12:01:03 +0800 Subject: [PATCH 1/6] [fix](complex) Support literal defaults for complex columns Issue Number: None Related PR: None Problem Summary: ARRAY, MAP, and STRUCT columns previously only supported DEFAULT NULL or very limited empty ARRAY defaults. This prevented valid literal defaults such as [], {}, [1, 2], {"a": 1}, or {1, "x"} from being used consistently when creating a table, adding a column, or reading default values from older rowsets. This change validates non-null complex defaults as type-appropriate literals in FE, teaches BE complex SerDes to parse these literal strings into typed Fields, and lets default iterators and direct schema change parse explicit defaults through non-nullable SerDes so type-mismatched nested values fail instead of being silently converted to NULL. Explicit nested NULL values are now accepted only when the nested SerDe is nullable, so non-nullable nested struct fields reject NULL defaults during parsing instead of reaching column insertion as a null Field. The regression coverage avoids asserting local-storage rewrite cancellation in cloud mode, where add-column schema change may finish as a metadata-only operation. INSERT fills unmentioned columns through a FE-side cast of the default literal, which is non-strict by default, so the regression exercises BE default parsing by reading rows written before a column with an invalid default was added through light schema change. JSON and VARIANT columns still only support DEFAULT NULL. Support literal default values for ARRAY, MAP, and STRUCT columns. - Test: - Unit Test: ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest - Unit Test: ./run-be-ut.sh --run --filter=DataTypeSerDeFromStringTest.*FromFeString -j 32 - Unit Test: ./run-be-ut.sh --run --filter=DataTypeSerDeFromStringTest.structFromFeString -j 32 - Unit Test: ./run-be-ut.sh --run --filter=DataTypeSerDeFromStringTest.*:ColumnReaderTest.DefaultValue* -j 64 - Unit Test: ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest,org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest - Build: ./build.sh --be - Build: ninja -C be/ut_build_ASAN src/storage/CMakeFiles/Storage.dir/schema_change/schema_change.cpp.o - Build and Regression test: doris-local-regression --network 10.26.20.3/24 all -d datatype_p0/complex_types -s test_complex_default_value - Format: build-support/clang-format.sh; build-support/check-format.sh; git diff --check - Regression test: doris-local-regression --network 10.26.20.3/24 run -d datatype_p0/complex_types -s test_complex_default_value -forceGenOut - Regression test: doris-local-regression --network 10.26.20.3/24 run -d datatype_p0/complex_types -s test_complex_default_value - Regression test: doris-local-regression --network 10.26.20.3/24 run -d schema_change_p0 -s test_dup_schema_key_add - Regression test: doris-local-regression --network 10.26.20.3/24 run -d schema_change_p0 -s test_unique_schema_key_change_add - Regression test: doris-local-regression --network 10.26.20.3/24 run -d schema_change_p0 -s test_modify_struct - Regression test (after rebase): doris-local-regression run -d datatype_p0/complex_types -s test_complex_default_value; run -d schema_change_p0 -s test_dup_schema_key_add,test_unique_schema_key_change_add,test_modify_struct - Regression test: doris-local-regression --network 10.26.20.3/24 run -d external_table_p0/iceberg -s test_iceberg_schema_change_complex_types - Behavior changed: Yes. ARRAY, MAP, and STRUCT non-null literal defaults are accepted and materialized; invalid explicit defaults are rejected in both default-iterator and direct schema-change materialization paths. JSON and VARIANT non-null defaults remain rejected. - Does this need documentation: No --- be/src/core/column/column_struct.cpp | 4 + .../complex_type_deserialize_util.h | 33 ++- .../data_type_serde/data_type_array_serde.cpp | 36 +++ .../data_type_serde/data_type_array_serde.h | 2 + .../data_type_serde/data_type_map_serde.cpp | 62 ++++ .../data_type_serde/data_type_map_serde.h | 2 + .../data_type_struct_serde.cpp | 81 +++++ .../data_type_serde/data_type_struct_serde.h | 2 + .../storage/schema_change/schema_change.cpp | 8 +- be/src/storage/segment/segment.cpp | 23 +- .../data_type_from_string_test.cpp | 87 +++++- .../plans/commands/info/ColumnDefinition.java | 79 ++++- .../plans/commands/AlterTableCommandTest.java | 2 +- .../commands/info/ColumnDefinitionTest.java | 57 ++++ .../test_complex_default_value.out | 13 + .../test_complex_default_value.groovy | 280 ++++++++++++++++++ ...iceberg_schema_change_complex_types.groovy | 4 +- .../test_dup_schema_key_add.groovy | 2 +- .../test_modify_struct.groovy | 6 +- .../test_unique_schema_key_change_add.groovy | 2 +- 20 files changed, 741 insertions(+), 44 deletions(-) create mode 100644 regression-test/data/datatype_p0/complex_types/test_complex_default_value.out create mode 100644 regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy diff --git a/be/src/core/column/column_struct.cpp b/be/src/core/column/column_struct.cpp index fb15785df8bb2d..1a4455a2079025 100644 --- a/be/src/core/column/column_struct.cpp +++ b/be/src/core/column/column_struct.cpp @@ -120,6 +120,10 @@ void ColumnStruct::insert(const Field& x) { DCHECK_EQ(x.get_type(), PrimitiveType::TYPE_STRUCT); const auto& tuple = x.get(); const size_t tuple_size = columns.size(); + if (tuple.empty() && tuple_size != 0) { + insert_default(); + return; + } if (tuple.size() != tuple_size) { throw doris::Exception( ErrorCode::INTERNAL_ERROR, diff --git a/be/src/core/data_type_serde/complex_type_deserialize_util.h b/be/src/core/data_type_serde/complex_type_deserialize_util.h index b1320b4b073fc3..2792d7e8044c36 100644 --- a/be/src/core/data_type_serde/complex_type_deserialize_util.h +++ b/be/src/core/data_type_serde/complex_type_deserialize_util.h @@ -17,6 +17,7 @@ #pragma once +#include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_serde.h" #include "core/string_ref.h" @@ -80,13 +81,37 @@ struct ComplexTypeDeserializeUtil { static bool is_null_string(const StringRef& str) { if (str.size == 4) { - // null - return str.data[0] == 'n' && str.data[1] == 'u' && str.data[2] == 'l' && - str.data[3] == 'l'; + // SQL NULL literal is case-insensitive. + return (str.data[0] == 'n' || str.data[0] == 'N') && + (str.data[1] == 'u' || str.data[1] == 'U') && + (str.data[2] == 'l' || str.data[2] == 'L') && + (str.data[3] == 'l' || str.data[3] == 'L'); } return false; } + static Status process_field(const DataTypeSerDeSPtr& serde, StringRef str, Field& field) { + str = str.trim_whitespace(); + auto nullable_serde = std::dynamic_pointer_cast(serde); + if (is_null_string(str)) { + if (nullable_serde == nullptr) { + return Status::InvalidArgument( + "NULL default is not allowed for non-nullable complex field"); + } + field = Field::create_field(Null {}); + return Status::OK(); + } + auto str_without_quote = str.trim_quote(); + auto nested_serde = serde; + if (nullable_serde != nullptr) { + // Complex default values need strict nested conversion. DataTypeNullableSerDe + // converts nested parse failures to NULL, which would silently accept + // type-mismatched default literals. + nested_serde = nullable_serde->get_nested_serde(); + } + return nested_serde->from_fe_string(str_without_quote.to_string(), field); + } + template static Status process_column(const DataTypeSerDeSPtr& serde, IColumn& column, StringRef& str, const DataTypeSerDe::FormatOptions& options) { @@ -107,4 +132,4 @@ struct ComplexTypeDeserializeUtil { } }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index c3980f58821ee5..93d5582fc1cb78 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -593,6 +593,42 @@ Status DataTypeArraySerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeArraySerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '[') { + return Status::InvalidArgument("Array does not start with '[' character, found '{}'", + slice.to_string()); + } + if (slice.back() != ']') { + return Status::InvalidArgument("Array does not end with ']' character, found '{}'", + slice.to_string()); + } + + Array array; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter( + slice, [&](char c) { return c == options.collection_delim; }); + array.reserve(split_result.size()); + for (auto& item : split_result) { + Field item_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + nested_serde, item.element, item_field)); + array.push_back(std::move(item_field)); + } + } + } + field = Field::create_field(std::move(array)); + return Status::OK(); +} + Status DataTypeArraySerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index 31378a61081ede..d3d3413d3eb705 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -54,6 +54,8 @@ class DataTypeArraySerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index eb5e16f07700ad..8596ca3c6b0a18 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -756,6 +756,68 @@ Status DataTypeMapSerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeMapSerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '{') { + std::stringstream ss; + ss << slice.front() << '\''; + return Status::InvalidArgument("Map does not start with '{' character, found '" + ss.str()); + } + if (slice.back() != '}') { + std::stringstream ss; + ss << slice.back() << '\''; + return Status::InvalidArgument("Map does not end with '}' character, found '" + ss.str()); + } + + Array keys; + Array values; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) { + return c == options.map_key_delim || c == options.collection_delim; + }); + if (split_result.size() % 2 != 0) { + return Status::InvalidArgument("Map does not have even number of key-value pairs"); + } + keys.reserve(split_result.size() / 2); + values.reserve(split_result.size() / 2); + for (int i = 0; i < split_result.size(); i += 2) { + if (split_result[i].delimiter != options.map_key_delim) { + return Status::InvalidArgument( + "Map key-value pair does not have map key delimiter"); + } + if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Map key-value pair does not have collection delimiter"); + } + + Field key_field; + Field value_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + key_serde, split_result[i].element, key_field)); + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + value_serde, split_result[i + 1].element, value_field)); + keys.push_back(std::move(key_field)); + values.push_back(std::move(value_field)); + } + } + } + + Map map; + map.reserve(2); + map.push_back(Field::create_field(std::move(keys))); + map.push_back(Field::create_field(std::move(values))); + field = Field::create_field(std::move(map)); + return Status::OK(); +} + Status DataTypeMapSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index bd6560c053c102..44bd9634eeca5d 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -47,6 +47,8 @@ class DataTypeMapSerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index a295dc2799fc93..3ea1c221e8d743 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -734,6 +734,87 @@ Status DataTypeStructSerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeStructSerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '{') { + std::stringstream ss; + ss << slice.front() << '\''; + return Status::InvalidArgument("Struct does not start with '{' character, found '" + + ss.str()); + } + if (slice.back() != '}') { + std::stringstream ss; + ss << slice.back() << '\''; + return Status::InvalidArgument("Struct does not end with '}' character, found '" + + ss.str()); + } + + Struct struct_field; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) { + return c == options.map_key_delim || c == options.collection_delim; + }); + + const auto elem_size = elem_serdes_ptrs.size(); + std::vector field_values; + field_values.reserve(elem_size); + if (split_result.size() == elem_size) { + for (int i = 0; i < split_result.size(); i++) { + if (i != split_result.size() - 1 && + split_result[i].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Struct field value {} is not separated by collection_delim.", i); + } + field_values.push_back(split_result[i].element); + } + } else if (split_result.size() == 2 * elem_size) { + int field_pos = 0; + for (int i = 0; i < split_result.size(); i += 2) { + if (split_result[i].delimiter != options.map_key_delim) { + return Status::InvalidArgument( + "Struct name-value pair does not have map key delimiter"); + } + if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) { + return Status::InvalidArgument( + "Struct name-value pair does not have collection delimiter"); + } + auto field_name = split_result[i].element.trim_quote(); + if (!field_name.eq(StringRef(elem_names[field_pos]))) { + return Status::InvalidArgument( + "Cannot find struct field name {} in schema.", + split_result[i].element.to_string()); + } + field_values.push_back(split_result[i + 1].element); + field_pos++; + } + } else { + return Status::InvalidArgument( + "Struct field number {} is not equal to schema field number {}.", + split_result.size(), elem_size); + } + + struct_field.reserve(elem_size); + for (int field_pos = 0; field_pos < elem_size; ++field_pos) { + Field elem_field; + RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field( + elem_serdes_ptrs[field_pos], field_values[field_pos], elem_field)); + struct_field.push_back(std::move(elem_field)); + } + } + } + + field = Field::create_field(std::move(struct_field)); + return Status::OK(); +} + Status DataTypeStructSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { return _from_string(str, column, options); diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index 1cfa3ee6c74db7..6910fc7cddd4c7 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -47,6 +47,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { Status from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const override; + Status from_fe_string(const std::string& str, Field& field) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, FormatOptions& options) const override; diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 6295091eb942ef..2cd9ab88db8bbb 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -44,6 +44,7 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_nullable.h" +#include "core/data_type/data_type_nullable.h" #include "exec/common/util.hpp" #include "exec/common/variant_util.h" #include "exprs/aggregate/aggregate_function.h" @@ -1528,8 +1529,11 @@ Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value) { if (!column_schema.is_nullable() || value.length() != 0) { - RETURN_IF_ERROR(column_schema.get_vec_type()->get_serde()->from_fe_string( - value, column_mapping->default_value)); + auto type = column_schema.get_vec_type(); + if (value.length() != 0) { + type = remove_nullable(type); + } + RETURN_IF_ERROR(type->get_serde()->from_fe_string(value, column_mapping->default_value)); } return Status::OK(); diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index af4ec1268a1d1b..536b7baa366db4 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -894,29 +894,22 @@ Status Segment::get_default_value_field(const TabletColumn& tablet_column, Field return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - if (default_value != "[]") { - return Status::NotSupported("Array default {} is unsupported", default_value); - } + // [] and {} need no nested SerDe, so an ARRAY/MAP TabletColumn built without sub-columns can + // still materialize an empty default. + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY && default_value == "[]") { *field = Field::create_field(Array {}); return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - return Status::NotSupported("STRUCT default type is unsupported"); - } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP) { - if (default_value != "{}") { - return Status::NotSupported("Map default {} is unsupported", default_value); - } + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP && default_value == "{}") { *field = Field::create_field(Map {Field::create_field(Array {}), Field::create_field(Array {})}); return Status::OK(); } - const auto serde = DataTypeFactory::instance() - .create_data_type(tablet_column.type(), tablet_column.precision(), - tablet_column.frac(), tablet_column.length()) - ->get_serde(); + // Any other non-null default, including a non-empty ARRAY/MAP or any STRUCT literal, needs the + // SerDe of the declared (possibly nested) type to parse recursively; a bare FieldType cannot + // describe nested item types. + const auto serde = remove_nullable(tablet_column.get_vec_type())->get_serde(); return serde->from_fe_string(default_value, *field); } diff --git a/be/test/core/data_type_serde/data_type_from_string_test.cpp b/be/test/core/data_type_serde/data_type_from_string_test.cpp index 7eeff50fbbe3c4..cacb46af93a627 100644 --- a/be/test/core/data_type_serde/data_type_from_string_test.cpp +++ b/be/test/core/data_type_serde/data_type_from_string_test.cpp @@ -35,6 +35,7 @@ #include "core/data_type/data_type_time.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/field.h" #include "core/string_ref.h" #include "gtest/gtest.h" #include "testutil/mock/mock_runtime_state.h" @@ -108,6 +109,90 @@ TEST_F(DataTypeSerDeFromStringTest, array) { } } +TEST_F(DataTypeSerDeFromStringTest, arrayFromFeString) { + auto type = std::make_shared( + std::make_shared(std::make_shared())); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"([1, NULL, nUlL, 3])", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_ARRAY); + const auto& array = field.get(); + ASSERT_EQ(array.size(), 4); + EXPECT_EQ(array[0].get(), 1); + EXPECT_TRUE(array[1].is_null()); + EXPECT_TRUE(array[2].is_null()); + EXPECT_EQ(array[3].get(), 3); + + EXPECT_FALSE(serde->from_fe_string(R"({"not_array": 1})", field).ok()); +} + +TEST_F(DataTypeSerDeFromStringTest, mapFromFeString) { + auto type = std::make_shared( + std::make_shared(), + std::make_shared(std::make_shared())); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"({"a": 1, "b": NULL, "c": nUlL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_MAP); + const auto& map = field.get(); + ASSERT_EQ(map.size(), 2); + ASSERT_EQ(map[0].get_type(), TYPE_ARRAY); + ASSERT_EQ(map[1].get_type(), TYPE_ARRAY); + + const auto& keys = map[0].get(); + const auto& values = map[1].get(); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(keys[0].get(), "a"); + EXPECT_EQ(values[0].get(), 1); + EXPECT_EQ(keys[1].get(), "b"); + EXPECT_TRUE(values[1].is_null()); + EXPECT_EQ(keys[2].get(), "c"); + EXPECT_TRUE(values[2].is_null()); + + EXPECT_FALSE(serde->from_fe_string(R"({"a": 1, "b"})", field).ok()); +} + +TEST_F(DataTypeSerDeFromStringTest, structFromFeString) { + DataTypes elems {std::make_shared(std::make_shared()), + std::make_shared(), + std::make_shared(std::make_shared())}; + Strings names {"f1", "f2", "f3"}; + auto type = std::make_shared(elems, names); + auto serde = type->get_serde(); + + Field field; + ASSERT_TRUE(serde->from_fe_string(R"({NULL, "x", nUlL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_STRUCT); + const auto& tuple_values = field.get(); + ASSERT_EQ(tuple_values.size(), 3); + EXPECT_TRUE(tuple_values[0].is_null()); + EXPECT_EQ(tuple_values[1].get(), "x"); + EXPECT_TRUE(tuple_values[2].is_null()); + + ASSERT_TRUE(serde->from_fe_string(R"({"f1": 7, "f2": "y", "f3": NULL})", field).ok()); + ASSERT_EQ(field.get_type(), TYPE_STRUCT); + const auto& named_values = field.get(); + ASSERT_EQ(named_values.size(), 3); + EXPECT_EQ(named_values[0].get(), 7); + EXPECT_EQ(named_values[1].get(), "y"); + EXPECT_TRUE(named_values[2].is_null()); + + EXPECT_FALSE(serde->from_fe_string(R"({1, 2})", field).ok()); + EXPECT_FALSE(serde->from_fe_string(R"({"f2": 1, "f1": "x", "f3": 3})", field).ok()); + + DataTypes non_nullable_elems {std::make_shared()}; + Strings non_nullable_names {"f1"}; + auto non_nullable_type = + std::make_shared(non_nullable_elems, non_nullable_names); + auto non_nullable_serde = non_nullable_type->get_serde(); + auto status = non_nullable_serde->from_fe_string(R"({NULL})", field); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("NULL default is not allowed") != std::string::npos); +} + TEST_F(DataTypeSerDeFromStringTest, bitmap) { auto type = std::make_shared(); auto serde = type->get_serde(); @@ -673,4 +758,4 @@ TEST_F(DataTypeSerDeFromStringTest, timeTest) { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index ae06f269cca17b..57a3616fba312d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -28,6 +28,11 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -538,17 +543,11 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); - } else if (type.isArrayType() && defaultValue.isPresent() && isOlap - && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get() - .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { - throw new AnalysisException("Array type column default value only support null or " - + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); } else { validateComplexTypeDefaultValue(); } - if (!isNullable && defaultValue.isPresent() - && defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { + if (!isNullable && hasNullDefaultValue()) { throw new AnalysisException( "Can not set null default value to non nullable column: " + name); } @@ -630,16 +629,68 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c * Validate non-null defaults for complex types before connector-specific validation. */ public void validateComplexTypeDefaultValue() throws AnalysisException { - if (!defaultValue.isPresent() || defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { - return; - } - if (type.isMapType()) { - throw new AnalysisException("Map type column default value just support null"); + if (type.isArrayType()) { + validateArrayDefaultValue(); + } else if (type.isMapType()) { + validateMapDefaultValue(); } else if (type.isStructType()) { - throw new AnalysisException("Struct type column default value just support null"); + validateStructDefaultValue(); } else if (type.isJsonType() || type.isVariantType()) { - throw new AnalysisException("Json or Variant type column default value just support null"); + if (hasNonNullDefaultValue()) { + throw new AnalysisException("Json or Variant type column default value only supports DEFAULT NULL"); + } + } + } + + private void validateArrayDefaultValue() { + if (!hasNonNullDefaultValue()) { + return; } + if (!isLiteralDefaultValue(ArrayLiteral.class)) { + throw new AnalysisException("Array type column default value only supports array literals or DEFAULT NULL"); + } + } + + private void validateMapDefaultValue() { + if (!hasNonNullDefaultValue()) { + return; + } + if (!isLiteralDefaultValue(MapLiteral.class)) { + throw new AnalysisException("Map type column default value only supports map literals or DEFAULT NULL"); + } + } + + private void validateStructDefaultValue() { + if (!hasNonNullDefaultValue()) { + return; + } + String value = defaultValue.get().getValue().trim(); + if (!isEmptyStructLiteral(value) && !isLiteralDefaultValue(StructLiteral.class)) { + throw new AnalysisException( + "Struct type column default value only supports struct literals or DEFAULT NULL"); + } + } + + private boolean isEmptyStructLiteral(String value) { + return value.startsWith("{") && value.endsWith("}") + && value.substring(1, value.length() - 1).trim().isEmpty(); + } + + private boolean isLiteralDefaultValue(Class literalClass) { + try { + Expression expression = new NereidsParser().parseExpression(defaultValue.get().getValue()); + return literalClass.isInstance(expression); + } catch (Exception e) { + return false; + } + } + + private boolean hasNonNullDefaultValue() { + return defaultValue.isPresent() && defaultValue.get().getValue() != null; + } + + private boolean hasNullDefaultValue() { + return defaultValue.isPresent() && defaultValue.get().getValue() == null; } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index ace25f884b30cc..f9bf211e5aada4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -316,7 +316,7 @@ void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisExcepti () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( "ALTER TABLE t MODIFY COLUMN st STRUCT DEFAULT 'x'").getOps())); Assertions.assertTrue(complexDefaultException.getMessage() - .contains("Struct type column default value just support null")); + .contains("Struct type column default value only supports struct literals or DEFAULT NULL")); AnalysisException onUpdateException = Assertions.assertThrows(AnalysisException.class, () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index 5a919c81a6d1e2..f0760f4a8c2938 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -22,12 +22,18 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.types.AggStateType; +import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.HllType; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.JsonType; +import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.QuantileStateType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.UuidType; +import org.apache.doris.nereids.types.VariantType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -36,6 +42,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Arrays; +import java.util.Collections; import java.util.Optional; public class ColumnDefinitionTest { @@ -152,4 +160,53 @@ public void testAddColumnRejectsUuidDynamicDefaults() { Optional.of(new DefaultValue("00112233-4455-6677-8899-aabbccddeeff")), ""); Assertions.assertFalse(literal.hasUuidDefaultValue()); } + + @Test + public void testComplexTypeLiteralDefaultValue() { + assertAllowsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "[]"); + assertAllowsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "[1, 2]"); + assertRejectsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "{}", + "only supports array literals or DEFAULT NULL"); + + assertAllowsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "{}"); + assertAllowsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "{\"a\": 1}"); + assertRejectsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "[]", + "only supports map literals or DEFAULT NULL"); + + StructType structType = new StructType(Arrays.asList( + new StructField("f1", IntegerType.INSTANCE, true, ""), + new StructField("f2", StringType.INSTANCE, true, ""))); + assertAllowsDefaultValue(structType, "{}"); + assertAllowsDefaultValue(structType, "{1, \"a\"}"); + assertRejectsDefaultValue(structType, "{\"f1\": 1, \"f2\": \"a\"}", + "only supports struct literals or DEFAULT NULL"); + + assertRejectsDefaultValue(JsonType.INSTANCE, "{}", "only supports DEFAULT NULL"); + assertRejectsDefaultValue(VariantType.INSTANCE, "{}", "only supports DEFAULT NULL"); + + Assertions.assertDoesNotThrow(() -> newColumnDefinition( + ArrayType.of(IntegerType.INSTANCE), Optional.empty()).validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + Assertions.assertDoesNotThrow(() -> newColumnDefinition( + MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), + Optional.of(DefaultValue.NULL_DEFAULT_VALUE)).validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + } + + private void assertAllowsDefaultValue(DataType type, String defaultValue) { + Assertions.assertDoesNotThrow(() -> newColumnDefinition( + type, Optional.of(new DefaultValue(defaultValue))).validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + } + + private void assertRejectsDefaultValue(DataType type, String defaultValue, String message) { + ColumnDefinition columnDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(defaultValue))); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> columnDefinition.validate( + true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + Assertions.assertTrue(exception.getMessage().contains(message)); + } + + private ColumnDefinition newColumnDefinition(DataType type, Optional defaultValue) { + return new ColumnDefinition("col1", type, false, null, true, defaultValue, ""); + } } diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out new file mode 100644 index 00000000000000..1fcd4594c345c2 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !literal_default -- +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} + +-- !null_default -- +1 \N \N \N \N \N + +-- !alter_default -- +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} + +-- !alter_default_after_insert -- +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} +2 [3, 4] {"z":9} {"f1":8, "f2":"y"} diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy new file mode 100644 index 00000000000000..ddbe5c2c047ed7 --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_complex_default_value") { + sql "DROP TABLE IF EXISTS test_complex_default_value_literal" + sql "DROP TABLE IF EXISTS test_complex_default_value_null" + sql "DROP TABLE IF EXISTS test_complex_default_value_alter_base" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_array_elem" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_map_key" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_map_value" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_struct_field" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_array_elem" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_map_key" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_struct_field" + + def waitForCancelledSchemaChange = { String tableName -> + def alterResult = [] + for (int i = 0; i < 600; ++i) { + alterResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1""" + if (alterResult.size() > 0 && + (alterResult[0][9] == "FINISHED" || alterResult[0][9] == "CANCELLED")) { + break + } + sleep(1000) + } + + logger.info("schema change result for ${tableName}: ${alterResult}") + assertTrue(alterResult.size() > 0) + assertEquals("CANCELLED", alterResult[0][9]) + assertTrue(alterResult[0].toString().contains("parse number fail")) + sql "SYNC" + } + + test { + sql """ + CREATE TABLE test_complex_default_value_bad_array ( + k INT, + v ARRAY DEFAULT '{}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception "only supports array literals or DEFAULT NULL" + } + + test { + sql """ + CREATE TABLE test_complex_default_value_bad_map ( + k INT, + v MAP DEFAULT '[]' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception "only supports map literals or DEFAULT NULL" + } + + test { + sql """ + CREATE TABLE test_complex_default_value_bad_json ( + k INT, + v JSON DEFAULT '{}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception "only supports DEFAULT NULL" + } + + test { + sql """ + CREATE TABLE test_complex_default_value_bad_variant ( + k INT, + v VARIANT DEFAULT '{}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception "only supports DEFAULT NULL" + } + + // INSERT fills unmentioned columns through a FE-side cast of the default literal, which is + // non-strict by default, so BE default parsing is exercised by reading rows that were written + // before the column with the invalid default was added by light schema change. + def checkInvalidDefaultOnRead = { String tableName, String columnDef -> + sql """ + CREATE TABLE ${tableName} ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='true' + ) + """ + sql "INSERT INTO ${tableName} VALUES (1)" + sql "ALTER TABLE ${tableName} ADD COLUMN v ${columnDef}" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1""" + time 600 + } + test { + sql "SELECT k, v FROM ${tableName}" + exception "parse number fail" + } + } + + checkInvalidDefaultOnRead.call("test_complex_default_value_bad_array_elem", + """ARRAY DEFAULT '["bad"]'""") + checkInvalidDefaultOnRead.call("test_complex_default_value_bad_map_key", + """MAP DEFAULT '{"bad": 1}'""") + checkInvalidDefaultOnRead.call("test_complex_default_value_bad_map_value", + """MAP DEFAULT '{"bad": "value"}'""") + checkInvalidDefaultOnRead.call("test_complex_default_value_bad_struct_field", + """STRUCT DEFAULT '{"bad"}'""") + + // Cloud mode may finish add-column schema change as a metadata-only operation, while local + // storage rewrites existing rows and cancels when the invalid default is materialized. + if (!isCloudMode()) { + sql """ + CREATE TABLE test_complex_default_value_bad_alter_array_elem ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='false' + ) + """ + + sql "INSERT INTO test_complex_default_value_bad_alter_array_elem VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_bad_alter_array_elem + ADD COLUMN v ARRAY DEFAULT '["bad"]' + """ + waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_array_elem") + + sql """ + CREATE TABLE test_complex_default_value_bad_alter_map_key ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='false' + ) + """ + + sql "INSERT INTO test_complex_default_value_bad_alter_map_key VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_bad_alter_map_key + ADD COLUMN v MAP DEFAULT '{"bad": 1}' + """ + waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_map_key") + + sql """ + CREATE TABLE test_complex_default_value_bad_alter_struct_field ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='false' + ) + """ + + sql "INSERT INTO test_complex_default_value_bad_alter_struct_field VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_bad_alter_struct_field + ADD COLUMN v STRUCT DEFAULT '{"bad"}' + """ + waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_struct_field") + } + + sql """ + CREATE TABLE test_complex_default_value_literal ( + k INT, + arr_empty ARRAY DEFAULT '[]', + arr_literal ARRAY DEFAULT '[1, 2]', + map_empty MAP DEFAULT '{}', + map_literal MAP DEFAULT '{"a": 10, "b": 20}', + struct_empty STRUCT DEFAULT '{}', + struct_literal STRUCT DEFAULT '{7, "x"}', + arr_nested_null ARRAY DEFAULT '[NULL, nUlL, 5]', + map_nested_null MAP DEFAULT '{"upper": NULL, "mixed": nUlL, "value": 6}', + struct_nested_null STRUCT DEFAULT '{NULL, NuLl, 7}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + + sql "INSERT INTO test_complex_default_value_literal(k) VALUES (1)" + order_qt_literal_default """ + SELECT k, arr_empty, arr_literal, map_empty, map_literal, struct_empty, struct_literal, + arr_nested_null, map_nested_null, struct_nested_null + FROM test_complex_default_value_literal + ORDER BY k + """ + + sql """ + CREATE TABLE test_complex_default_value_null ( + k INT, + arr_col ARRAY DEFAULT NULL, + map_col MAP DEFAULT NULL, + struct_col STRUCT DEFAULT NULL, + json_col JSON DEFAULT NULL, + variant_col VARIANT DEFAULT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + + sql "INSERT INTO test_complex_default_value_null(k) VALUES (1)" + order_qt_null_default """ + SELECT k, arr_col, map_col, struct_col, json_col, variant_col + FROM test_complex_default_value_null + ORDER BY k + """ + + sql """ + CREATE TABLE test_complex_default_value_alter_base ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='true' + ) + """ + + sql "INSERT INTO test_complex_default_value_alter_base VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_alter_base + ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', + ADD COLUMN map_added MAP NOT NULL DEFAULT '{"z": 9}', + ADD COLUMN struct_added STRUCT NOT NULL DEFAULT '{8, "y"}' + """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_base' ORDER BY createtime DESC LIMIT 1""" + time 600 + } + + order_qt_alter_default """ + SELECT k, arr_added, map_added, struct_added + FROM test_complex_default_value_alter_base + ORDER BY k + """ + + sql "INSERT INTO test_complex_default_value_alter_base(k) VALUES (2)" + order_qt_alter_default_after_insert """ + SELECT k, arr_added, map_added, struct_added + FROM test_complex_default_value_alter_base + ORDER BY k + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy index b79f9f1135de84..9045b4a7e2fc33 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_change_complex_types.groovy @@ -77,7 +77,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do STRUCT(1, CAST(1.1 AS FLOAT), 'abc', MAP(1, 10), ARRAY(1, 2)) )""" - // Complex type default value only supports NULL + // Invalid complex type default value is rejected. test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN st STRUCT< @@ -87,7 +87,7 @@ suite("test_iceberg_schema_change_complex_types", "p0,external,doris,external_do sm: MAP COMMENT 'm1', sa: ARRAY COMMENT 'a1' > DEFAULT 'x'""" - exception "just support null" + exception "only supports struct literals or DEFAULT NULL" } // Cannot change nullable complex column to not null diff --git a/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy b/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy index bcc5583a6fc56b..b2bb2ed20e633b 100644 --- a/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_schema_key_add.groovy @@ -587,7 +587,7 @@ suite("test_dup_schema_key_add","p0") { //TODO Test the duplicate model by adding a column with JSON type none default value - errorMessage="errCode = 2, detailMessage = Json or Variant type column default value just support null" + errorMessage="errCode = 2, detailMessage = Json or Variant type column default value only supports DEFAULT NULL" expectException({ sql initTable sql initTableData diff --git a/regression-test/suites/schema_change_p0/test_modify_struct.groovy b/regression-test/suites/schema_change_p0/test_modify_struct.groovy index 12effd00a15cc9..0a750b2bc21df4 100644 --- a/regression-test/suites/schema_change_p0/test_modify_struct.groovy +++ b/regression-test/suites/schema_change_p0/test_modify_struct.groovy @@ -306,10 +306,10 @@ suite ("test_modify_struct") { String[][] descRes = sql """ desc ${table_name} """ logger.info(descRes[2][2]) assertEquals(descRes[2][2].toString().toLowerCase(), "no") - // sc do not support new struct type with not null. - def changeNotNullErr = "Struct type column default value just support null" + // Invalid struct default value is rejected. + def changeNotNullErr = "Struct type column default value only supports struct literals or DEFAULT NULL" expectExceptionLike({ - sql """ alter table ${table_name} add column c_s_not_null1 struct NOT NULL default '{}'""" + sql """ alter table ${table_name} add column c_s_not_null1 struct NOT NULL default 'x'""" waitUntilSchemaChangeDone.call(table_name, "") }, changeNotNullErr) diff --git a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy index 7a39ab2b11dd2b..b462e0a4674b60 100644 --- a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy +++ b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_add.groovy @@ -256,7 +256,7 @@ suite("test_unique_schema_key_change_add","p0") { //TODO Test the unique model by adding a column with JSON type none default value - errorMessage="errCode = 2, detailMessage = Json or Variant type column default value just support null" + errorMessage="errCode = 2, detailMessage = Json or Variant type column default value only supports DEFAULT NULL" expectException({ sql initTable sql initTableData From 3303863286e2c1467f522154524605daccfd9f39 Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Fri, 9 Oct 2026 14:06:09 +0800 Subject: [PATCH 2/6] [fix](complex) Canonicalize complex column default literals in FE ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: FE validated an ARRAY/MAP/STRUCT default only by its literal shape and stored the raw SQL text, while BE parsed that text with the plain string-to-complex grammar. Values that are valid SQL literals but not plain text therefore read back wrong or failed late: `ARRAY DEFAULT '[DATEV2 "2024-01-01"]'` materialized `MIN_DATE_V2`, `ARRAY DEFAULT '[1e3]'` failed when old rows were read, `ARRAY DEFAULT '["a""b"]'` stored `a""b`, and `MAP DEFAULT '{"a":1,"a":2}'` kept both pairs although the FE literal keeps `{"a":2}`. `SHOW CREATE TABLE` also wrapped the raw default in double quotes without escaping, so `CREATE TABLE LIKE` could not replay a MAP default, and a one-argument `replace_value` load mapping on such a column hit the scalar-only precondition of `ColumnDef.validateDefaultValue`. FE now parses the default as a SQL literal, casts every nested value to the declared nested type with strict checking, and stores a canonical text (unquoted numbers, bool as 1/0, double quoted strings/dates/ips, nested brackets, NULL). Type mismatches, typed or exponent literals and repeated map keys are therefore resolved at DDL time, and BE only ever parses the canonical form through the default value iterator, schema change, and the INSERT cast. Nested string values that contain quotes or backslashes are rejected, because neither BE parser decodes escapes and the DDL parser keeps the default text verbatim. `Column.toSql()` wraps a default that contains double quotes in single quotes so SHOW CREATE TABLE output replays through CREATE TABLE LIKE, and load `default_value`/`replace_value` mappings validate complex columns through the same canonicalizer. ### Release note None ### Check List (For Author) - Test: Regression test / Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest,org.apache.doris.catalog.ColumnTest - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.commands.AlterTableCommandTest - ./run-be-ut.sh --run --filter='DataTypeSerDeFromStringTest.*' - doris-local-regression run -d datatype_p0/complex_types -s test_complex_default_value - doris-local-regression run -d schema_change_p0 -s test_dup_schema_key_add,test_unique_schema_key_change_add,test_modify_struct - Behavior changed: Yes. Complex column defaults are type checked at DDL time and stored in canonical form; nested string defaults may not contain quotes or backslashes. - Does this need documentation: No --- .../data_type_from_string_test.cpp | 43 +++ .../java/org/apache/doris/catalog/Column.java | 14 +- .../nereids/load/NereidsDataDescription.java | 20 +- .../plans/commands/info/ColumnDefinition.java | 61 +--- .../info/ComplexTypeDefaultValue.java | 175 ++++++++++ .../org/apache/doris/catalog/ColumnTest.java | 11 + .../commands/info/ColumnDefinitionTest.java | 68 +++- .../test_complex_default_value.out | 19 +- ...st_complex_default_value_replace_value.csv | 2 + .../test_complex_default_value.groovy | 317 ++++++++---------- 10 files changed, 476 insertions(+), 254 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java create mode 100644 regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv diff --git a/be/test/core/data_type_serde/data_type_from_string_test.cpp b/be/test/core/data_type_serde/data_type_from_string_test.cpp index cacb46af93a627..12f2ab19c22467 100644 --- a/be/test/core/data_type_serde/data_type_from_string_test.cpp +++ b/be/test/core/data_type_serde/data_type_from_string_test.cpp @@ -193,6 +193,49 @@ TEST_F(DataTypeSerDeFromStringTest, structFromFeString) { EXPECT_TRUE(status.to_string().find("NULL default is not allowed") != std::string::npos); } +// FE stores complex defaults in a canonical text: unquoted numbers (bool as 1/0), double quoted +// strings/dates/ips and nested brackets. These cases pin down what BE must accept from that text. +TEST_F(DataTypeSerDeFromStringTest, canonicalComplexDefaultFromFeString) { + auto check = [](const DataTypePtr& type, const std::string& text, const std::string& expected) { + Field field; + auto st = type->get_serde()->from_fe_string(text, field); + ASSERT_TRUE(st.ok()) << text << ": " << st.to_string(); + auto column = type->create_column(); + column->insert(field); + EXPECT_EQ(type->to_string(*column, 0), expected) << text; + }; + auto nullable = [](DataTypePtr type) { return std::make_shared(type); }; + + check(std::make_shared(nullable(std::make_shared())), + R"(["it's", "x,y", "[z]", "{k:v}", "null", "", NULL])", + R"(["it's", "x,y", "[z]", "{k:v}", "null", "", null])"); + check(std::make_shared(nullable(std::make_shared())), + R"(["2024-01-01"])", R"(["2024-01-01"])"); + check(std::make_shared(nullable(std::make_shared(0))), + R"(["2024-01-01 10:20:30"])", R"(["2024-01-01 10:20:30"])"); + check(std::make_shared(nullable(std::make_shared(3))), + R"(["2024-01-01 10:20:30.123"])", R"(["2024-01-01 10:20:30.123"])"); + check(std::make_shared(nullable(std::make_shared())), "[1, 0]", + "[1, 0]"); + check(std::make_shared(nullable(std::make_shared())), + "[1.0E10, 1.5]", "[10000000000, 1.5]"); + check(std::make_shared(nullable(std::make_shared(10, 2))), + "[1.23, 1000]", "[1.23, 1000.00]"); + check(std::make_shared(nullable(std::make_shared())), + R"(["127.0.0.1"])", R"(["127.0.0.1"])"); + check(std::make_shared(std::make_shared(), + nullable(std::make_shared( + nullable(std::make_shared())))), + R"({"a":[1, 2], "b":[], "c":NULL})", R"({"a":[1, 2], "b":[], "c":null})"); + check(std::make_shared( + DataTypes {nullable(std::make_shared()), + nullable(std::make_shared( + std::make_shared(), + nullable(std::make_shared())))}, + Strings {"f1", "f2"}), + R"({7, {"x":1}})", R"({"f1":7, "f2":{"x":1}})"); +} + TEST_F(DataTypeSerDeFromStringTest, bitmap) { auto type = std::make_shared(); auto serde = type->get_serde(); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index 6272cfc0a3be4b..e0b4ecfe43c3fb 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -890,6 +890,18 @@ public String toSql() { return toSql(false, false); } + /** + * The DDL parser keeps the text between the quotes of a default value verbatim, so a value that + * contains double quotes (complex type defaults store nested strings as "...") is wrapped in single + * quotes to let SHOW CREATE TABLE output be replayed by CREATE TABLE LIKE. + */ + private static String quoteDefaultValue(String value) { + if (value.indexOf('"') >= 0 && value.indexOf('\'') < 0) { + return "'" + value + "'"; + } + return "\"" + value + "\""; + } + public String toSql(boolean isUniqueTable) { return toSql(isUniqueTable, false); } @@ -930,7 +942,7 @@ public String toSql(boolean isUniqueTable, boolean isCompatible, boolean useGene if (defaultValueExprDef != null) { sb.append(" DEFAULT ").append(defaultValue).append(""); } else { - sb.append(" DEFAULT \"").append(defaultValue).append("\""); + sb.append(" DEFAULT ").append(quoteDefaultValue(defaultValue)); } } if ((getDataType() == PrimitiveType.BITMAP) && defaultValue != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java index 7c0c4b2fd97221..0022b4d206cf99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsDataDescription.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; @@ -49,6 +50,8 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.plans.commands.info.ComplexTypeDefaultValue; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileFormatType; @@ -599,10 +602,23 @@ private static void validateDefaultValue(List args, Column column) throw } if (args.get(0) != null) { - ColumnDef.validateDefaultValue(column.getOriginType(), args.get(0), column.getDefaultValueExprDef()); + validateColumnValue(column, args.get(0)); } } + private static void validateColumnValue(Column column, String value) throws AnalysisException { + Type type = column.getOriginType(); + if (type.isComplexType()) { + try { + ComplexTypeDefaultValue.canonicalize(DataType.fromCatalogType(type), value); + } catch (org.apache.doris.nereids.exceptions.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + return; + } + ColumnDef.validateDefaultValue(type, value, column.getDefaultValueExprDef()); + } + private static void validateMd5sum(List args, Map columnNameMap) throws AnalysisException { for (int i = 0; i < args.size(); ++i) { String argColumn = args.get(i); @@ -634,7 +650,7 @@ private static void validateReplaceValue(List args, Column column) throw } if (replaceValue != null) { - ColumnDef.validateDefaultValue(column.getOriginType(), replaceValue, column.getDefaultValueExprDef()); + validateColumnValue(column, replaceValue); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 57a3616fba312d..f7511126dcc9ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -28,11 +28,6 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; -import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; -import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -627,61 +622,17 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c /** * Validate non-null defaults for complex types before connector-specific validation. + * ARRAY, MAP and STRUCT defaults are rewritten into the canonical literal text that BE parses. */ public void validateComplexTypeDefaultValue() throws AnalysisException { - if (type.isArrayType()) { - validateArrayDefaultValue(); - } else if (type.isMapType()) { - validateMapDefaultValue(); - } else if (type.isStructType()) { - validateStructDefaultValue(); - } else if (type.isJsonType() || type.isVariantType()) { - if (hasNonNullDefaultValue()) { - throw new AnalysisException("Json or Variant type column default value only supports DEFAULT NULL"); - } - } - } - - private void validateArrayDefaultValue() { - if (!hasNonNullDefaultValue()) { - return; - } - if (!isLiteralDefaultValue(ArrayLiteral.class)) { - throw new AnalysisException("Array type column default value only supports array literals or DEFAULT NULL"); - } - } - - private void validateMapDefaultValue() { if (!hasNonNullDefaultValue()) { return; } - if (!isLiteralDefaultValue(MapLiteral.class)) { - throw new AnalysisException("Map type column default value only supports map literals or DEFAULT NULL"); - } - } - - private void validateStructDefaultValue() { - if (!hasNonNullDefaultValue()) { - return; - } - String value = defaultValue.get().getValue().trim(); - if (!isEmptyStructLiteral(value) && !isLiteralDefaultValue(StructLiteral.class)) { - throw new AnalysisException( - "Struct type column default value only supports struct literals or DEFAULT NULL"); - } - } - - private boolean isEmptyStructLiteral(String value) { - return value.startsWith("{") && value.endsWith("}") - && value.substring(1, value.length() - 1).trim().isEmpty(); - } - - private boolean isLiteralDefaultValue(Class literalClass) { - try { - Expression expression = new NereidsParser().parseExpression(defaultValue.get().getValue()); - return literalClass.isInstance(expression); - } catch (Exception e) { - return false; + if (type.isArrayType() || type.isMapType() || type.isStructType()) { + defaultValue = Optional.of(new DefaultValue( + ComplexTypeDefaultValue.canonicalize(type, defaultValue.get().getValue()))); + } else if (type.isJsonType() || type.isVariantType()) { + throw new AnalysisException("Json or Variant type column default value only supports DEFAULT NULL"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java new file mode 100644 index 00000000000000..421af8ad72c638 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; + +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; + +/** + * Validates a non-null ARRAY/MAP/STRUCT column default and rewrites it into a canonical literal text. + * + *

The user supplied text is parsed as a SQL literal and every nested value is cast to the declared + * nested type, so a type mismatch is rejected at DDL time instead of when old rows are read. The + * canonical text only contains plain nested values (unquoted numbers, double quoted strings, nested + * brackets and NULL). BE parses the stored text in two places that must agree: the complex SerDe + * {@code from_fe_string} used by the default value iterator and schema change, and the + * string-to-complex cast used when INSERT fills an unmentioned column. Neither of them decodes + * escape sequences, and the DDL parser keeps the text of a default value verbatim when SHOW CREATE + * TABLE output is replayed, so string values containing quotes or backslashes are rejected instead + * of stored. + */ +public class ComplexTypeDefaultValue { + private ComplexTypeDefaultValue() { + } + + /** + * Validate the default literal of a complex column and return its canonical text. + */ + public static String canonicalize(DataType type, String defaultValue) throws AnalysisException { + Preconditions.checkArgument(type.isArrayType() || type.isMapType() || type.isStructType(), + "%s is not a complex type", type); + Expression expression; + try { + expression = new NereidsParser().parseExpression(defaultValue); + } catch (Exception e) { + throw literalShapeException(type); + } + if (!hasLiteralShape(expression, type)) { + throw literalShapeException(type); + } + try { + return render((Literal) expression, type); + } catch (AnalysisException e) { + throw new AnalysisException(String.format("Invalid default value '%s' for %s column: %s", + defaultValue, type.toSql(), e.getMessage()), e); + } + } + + private static boolean hasLiteralShape(Expression expression, DataType type) { + if (type.isArrayType()) { + return expression instanceof ArrayLiteral; + } + if (type.isMapType()) { + return expression instanceof MapLiteral; + } + // `{}` parses as an empty map literal and means every struct field takes its own default. + return expression instanceof StructLiteral + || (expression instanceof MapLiteral && ((MapLiteral) expression).getValue().isEmpty()); + } + + private static AnalysisException literalShapeException(DataType type) { + String literalKind = type.isArrayType() ? "array" : type.isMapType() ? "map" : "struct"; + return new AnalysisException(String.format("%s type column default value only supports %s literals" + + " or DEFAULT NULL", capitalize(literalKind), literalKind)); + } + + private static String capitalize(String value) { + return Character.toUpperCase(value.charAt(0)) + value.substring(1); + } + + private static String render(Literal literal, DataType type) throws AnalysisException { + if (literal instanceof NullLiteral) { + return "NULL"; + } + if (type.isArrayType()) { + if (!(literal instanceof ArrayLiteral)) { + throw new AnalysisException(literal.toSql() + " is not an array literal"); + } + DataType itemType = ((ArrayType) type).getItemType(); + StringJoiner joiner = new StringJoiner(", ", "[", "]"); + for (Literal item : ((ArrayLiteral) literal).getValue()) { + joiner.add(render(item, itemType)); + } + return joiner.toString(); + } + if (type.isMapType()) { + if (!(literal instanceof MapLiteral)) { + throw new AnalysisException(literal.toSql() + " is not a map literal"); + } + MapType mapType = (MapType) type; + // The parser keeps the last value of a repeated key, so a repeated key is stored once. + StringJoiner joiner = new StringJoiner(", ", "{", "}"); + for (Map.Entry entry : ((MapLiteral) literal).getValue().entrySet()) { + joiner.add(render(entry.getKey(), mapType.getKeyType()) + ":" + + render(entry.getValue(), mapType.getValueType())); + } + return joiner.toString(); + } + if (type.isStructType()) { + // `{}` parses as an empty map literal and means every field takes its own default. + if (literal instanceof MapLiteral && ((MapLiteral) literal).getValue().isEmpty()) { + return "{}"; + } + if (!(literal instanceof StructLiteral)) { + throw new AnalysisException(literal.toSql() + " is not a struct literal"); + } + List fields = ((StructType) type).getFields(); + List values = ((StructLiteral) literal).getValue(); + if (values.size() != fields.size()) { + throw new AnalysisException(String.format("struct literal has %d fields but the column has %d", + values.size(), fields.size())); + } + StringJoiner joiner = new StringJoiner(", ", "{", "}"); + for (int i = 0; i < fields.size(); i++) { + joiner.add(render(values.get(i), fields.get(i).getDataType())); + } + return joiner.toString(); + } + return renderScalar((Literal) literal.checkedCastTo(type), type); + } + + private static String renderScalar(Literal literal, DataType type) throws AnalysisException { + if (type.isBooleanType()) { + return ((BooleanLiteral) literal).getValue() ? "1" : "0"; + } + if (type.isNumericType()) { + if (type.isFloatLikeType() && !Double.isFinite(((Number) literal.getValue()).doubleValue())) { + throw new AnalysisException(literal.toSql() + " is not a finite number"); + } + return literal.getStringValue(); + } + if (type.isStringLikeType() || type.isDateType() || type.isDateV2Type() || type.isDateTimeType() + || type.isDateTimeV2Type() || type.isIPType()) { + String value = literal.getStringValue(); + if (value.indexOf('"') >= 0 || value.indexOf('\'') >= 0 || value.indexOf('\\') >= 0) { + throw new AnalysisException("string value " + literal.toSql() + + " must not contain quote or backslash"); + } + return "\"" + value + "\""; + } + throw new AnalysisException(type.toSql() + " is not supported as a nested type of a default value"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java index d78c92d0213bab..0f1fac2d08aafb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java @@ -224,6 +224,17 @@ public void testStrictNestedPrimitivePromotionRules() { ScalarType.createDecimalV3Type(5, 2), ScalarType.createDecimalV3Type(10, 3))); } + @Test + public void testToSqlQuotesDefaultValueForReplay() { + Column plain = new Column("c1", Type.STRING, false, null, true, "abc", ""); + Assertions.assertTrue(plain.toSql().contains(" DEFAULT \"abc\""), plain.toSql()); + Column complex = new Column("c2", new MapType(Type.STRING, Type.INT), false, null, true, + "{\"a\":1}", ""); + Assertions.assertTrue(complex.toSql().contains(" DEFAULT '{\"a\":1}'"), complex.toSql()); + Column quoted = new Column("c3", Type.STRING, false, null, true, "it's", ""); + Assertions.assertTrue(quoted.toSql().contains(" DEFAULT \"it's\""), quoted.toSql()); + } + @Test public void testIcebergNestedDecimalPromotionRules() { Assertions.assertTrue(ColumnType.isSupportedIcebergNestedDecimalPromotion( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index f0760f4a8c2938..575f1a1bc1067a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -23,7 +23,11 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.HllType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.JsonType; @@ -163,23 +167,51 @@ public void testAddColumnRejectsUuidDynamicDefaults() { @Test public void testComplexTypeLiteralDefaultValue() { - assertAllowsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "[]"); - assertAllowsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "[1, 2]"); - assertRejectsDefaultValue(ArrayType.of(IntegerType.INSTANCE), "{}", - "only supports array literals or DEFAULT NULL"); - - assertAllowsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "{}"); - assertAllowsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "{\"a\": 1}"); - assertRejectsDefaultValue(MapType.of(StringType.INSTANCE, IntegerType.INSTANCE), "[]", - "only supports map literals or DEFAULT NULL"); + ArrayType intArray = ArrayType.of(IntegerType.INSTANCE); + assertCanonicalDefaultValue(intArray, "[]", "[]"); + assertCanonicalDefaultValue(intArray, "[1, 2]", "[1, 2]"); + assertCanonicalDefaultValue(intArray, "[NULL, nUlL, 5]", "[NULL, NULL, 5]"); + assertCanonicalDefaultValue(intArray, "[\"7\", 1e3]", "[7, 1000]"); + assertRejectsDefaultValue(intArray, "{}", "only supports array literals or DEFAULT NULL"); + assertRejectsDefaultValue(intArray, "[1 + 1]", "only supports array literals or DEFAULT NULL"); + assertRejectsDefaultValue(intArray, "[\"bad\"]", "Invalid default value '[\"bad\"]' for ARRAY"); + assertRejectsDefaultValue(intArray, "[[1]]", "Invalid default value"); + + assertCanonicalDefaultValue(ArrayType.of(DateV2Type.INSTANCE), + "[DATEV2 \"2024-01-01\", \"2024-02-02\"]", "[\"2024-01-01\", \"2024-02-02\"]"); + assertCanonicalDefaultValue(ArrayType.of(BooleanType.INSTANCE), "[true, false]", "[1, 0]"); + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[1.5, 2]", "[1.5, 2.0]"); + assertCanonicalDefaultValue(ArrayType.of(DecimalV3Type.createDecimalV3Type(10, 2)), "[1.234]", "[1.23]"); + assertCanonicalDefaultValue(ArrayType.of(StringType.INSTANCE), "['x,y', \"[z]\", \"{k:v}\", '']", + "[\"x,y\", \"[z]\", \"{k:v}\", \"\"]"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "[\"a\"\"b\"]", + "must not contain quote or backslash"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "['it''s']", + "must not contain quote or backslash"); + assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "['a\\\\b']", + "must not contain quote or backslash"); + assertCanonicalDefaultValue(ArrayType.of(intArray), "[[1], [], NULL]", "[[1], [], NULL]"); + + MapType stringIntMap = MapType.of(StringType.INSTANCE, IntegerType.INSTANCE); + assertCanonicalDefaultValue(stringIntMap, "{}", "{}"); + assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"b\": NULL}", "{\"a\":1, \"b\":NULL}"); + assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"a\": 2}", "{\"a\":2}"); + assertRejectsDefaultValue(stringIntMap, "[]", "only supports map literals or DEFAULT NULL"); + assertRejectsDefaultValue(stringIntMap, "{\"a\": \"bad\"}", "Invalid default value"); + assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"bad\": 1}", + "Invalid default value"); StructType structType = new StructType(Arrays.asList( new StructField("f1", IntegerType.INSTANCE, true, ""), new StructField("f2", StringType.INSTANCE, true, ""))); - assertAllowsDefaultValue(structType, "{}"); - assertAllowsDefaultValue(structType, "{1, \"a\"}"); + assertCanonicalDefaultValue(structType, "{}", "{}"); + assertCanonicalDefaultValue(structType, "{1, \"a\"}", "{1, \"a\"}"); + assertCanonicalDefaultValue(structType, "{NULL, 2}", "{NULL, \"2\"}"); assertRejectsDefaultValue(structType, "{\"f1\": 1, \"f2\": \"a\"}", "only supports struct literals or DEFAULT NULL"); + assertRejectsDefaultValue(structType, "{1}", "struct literal has 1 fields but the column has 2"); + assertRejectsDefaultValue(structType, "{\"bad\", \"a\"}", "Invalid default value"); + assertRejectsDefaultValue(structType, "[]", "only supports struct literals or DEFAULT NULL"); assertRejectsDefaultValue(JsonType.INSTANCE, "{}", "only supports DEFAULT NULL"); assertRejectsDefaultValue(VariantType.INSTANCE, "{}", "only supports DEFAULT NULL"); @@ -193,17 +225,21 @@ public void testComplexTypeLiteralDefaultValue() { true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); } - private void assertAllowsDefaultValue(DataType type, String defaultValue) { - Assertions.assertDoesNotThrow(() -> newColumnDefinition( - type, Optional.of(new DefaultValue(defaultValue))).validate( - true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); + private void assertCanonicalDefaultValue(DataType type, String defaultValue, String canonical) { + ColumnDefinition columnDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(defaultValue))); + columnDefinition.validate(true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS); + Assertions.assertEquals(canonical, columnDefinition.getDefaultValueString()); + // the canonical text is itself a valid default and is stable + ColumnDefinition canonicalDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(canonical))); + canonicalDefinition.validate(true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS); + Assertions.assertEquals(canonical, canonicalDefinition.getDefaultValueString()); } private void assertRejectsDefaultValue(DataType type, String defaultValue, String message) { ColumnDefinition columnDefinition = newColumnDefinition(type, Optional.of(new DefaultValue(defaultValue))); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> columnDefinition.validate( true, Collections.emptySet(), Collections.emptySet(), false, KeysType.DUP_KEYS)); - Assertions.assertTrue(exception.getMessage().contains(message)); + Assertions.assertTrue(exception.getMessage().contains(message), exception.getMessage()); } private ColumnDefinition newColumnDefinition(DataType type, Optional defaultValue) { diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out index 1fcd4594c345c2..27808c9f67f77b 100644 --- a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out @@ -1,13 +1,24 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !literal_default -- -1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] + +-- !like_default -- +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] -- !null_default -- 1 \N \N \N \N \N -- !alter_default -- -1 [3, 4] {"z":9} {"f1":8, "f2":"y"} +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] -- !alter_default_after_insert -- -1 [3, 4] {"z":9} {"f1":8, "f2":"y"} -2 [3, 4] {"z":9} {"f1":8, "f2":"y"} +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] +2 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] + +-- !alter_direct_default -- +1 [3, 4] {"a":2} {"f1":8, "f2":["2024-01-01"]} ["x,y", ""] + +-- !replace_value -- +1 {"a":1} +2 {"b":2} + diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv new file mode 100644 index 00000000000000..4d41303fa8fd31 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_replace_value.csv @@ -0,0 +1,2 @@ +1 \N +2 {"b":2} diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy index ddbe5c2c047ed7..e33f39e4ab04b9 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -17,181 +17,71 @@ suite("test_complex_default_value") { sql "DROP TABLE IF EXISTS test_complex_default_value_literal" + sql "DROP TABLE IF EXISTS test_complex_default_value_like" sql "DROP TABLE IF EXISTS test_complex_default_value_null" sql "DROP TABLE IF EXISTS test_complex_default_value_alter_base" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_array_elem" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_map_key" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_map_value" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_struct_field" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_array_elem" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_map_key" - sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter_struct_field" + sql "DROP TABLE IF EXISTS test_complex_default_value_alter_direct" + sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter" + sql "DROP TABLE IF EXISTS test_complex_default_value_replace_value" - def waitForCancelledSchemaChange = { String tableName -> - def alterResult = [] - for (int i = 0; i < 600; ++i) { - alterResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1""" - if (alterResult.size() > 0 && - (alterResult[0][9] == "FINISHED" || alterResult[0][9] == "CANCELLED")) { - break - } - sleep(1000) + def createTableRejects = { String columnDef, String message -> + test { + sql """ + CREATE TABLE test_complex_default_value_rejected ( + k INT, + v ${columnDef} + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + exception message } - - logger.info("schema change result for ${tableName}: ${alterResult}") - assertTrue(alterResult.size() > 0) - assertEquals("CANCELLED", alterResult[0][9]) - assertTrue(alterResult[0].toString().contains("parse number fail")) - sql "SYNC" - } - - test { - sql """ - CREATE TABLE test_complex_default_value_bad_array ( - k INT, - v ARRAY DEFAULT '{}' - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES('replication_num'='1') - """ - exception "only supports array literals or DEFAULT NULL" } + // the default must be a literal of the column's own shape + createTableRejects.call("ARRAY DEFAULT '{}'", "only supports array literals or DEFAULT NULL") + createTableRejects.call("ARRAY DEFAULT '[1 + 1]'", "only supports array literals or DEFAULT NULL") + createTableRejects.call("MAP DEFAULT '[]'", "only supports map literals or DEFAULT NULL") + createTableRejects.call("STRUCT DEFAULT '[]'", "only supports struct literals or DEFAULT NULL") + createTableRejects.call("""STRUCT DEFAULT '{"f1": 1, "f2": "a"}'""", "only supports struct literals or DEFAULT NULL") + createTableRejects.call("JSON DEFAULT '{}'", "only supports DEFAULT NULL") + createTableRejects.call("VARIANT DEFAULT '{}'", "only supports DEFAULT NULL") + + // every nested value is cast to the declared nested type at DDL time + createTableRejects.call("""ARRAY DEFAULT '["bad"]'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"bad": 1}'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"bad": "value"}'""", "Invalid default value") + createTableRejects.call("""STRUCT DEFAULT '{"bad"}'""", "Invalid default value") + createTableRejects.call("""STRUCT DEFAULT '{1}'""", "struct literal has 1 fields but the column has 2") + createTableRejects.call("""ARRAY> DEFAULT '[[1], ["bad"]]'""", "Invalid default value") + + // nested string values are stored as plain double quoted text, so quotes and backslashes are rejected + createTableRejects.call("""ARRAY DEFAULT '["a""b"]'""", "must not contain quote or backslash") + createTableRejects.call("""MAP DEFAULT '{"a\\\\\\\\b": 1}'""", "must not contain quote or backslash") + createTableRejects.call("""ARRAY DEFAULT '["it''s"]'""", "must not contain quote or backslash") + + // ADD COLUMN validates the default the same way, for both light and direct schema change + sql """ + CREATE TABLE test_complex_default_value_bad_alter ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1', 'light_schema_change'='false') + """ + sql "INSERT INTO test_complex_default_value_bad_alter VALUES (1)" test { - sql """ - CREATE TABLE test_complex_default_value_bad_map ( - k INT, - v MAP DEFAULT '[]' - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES('replication_num'='1') - """ - exception "only supports map literals or DEFAULT NULL" + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '["bad"]'""" + exception "Invalid default value" } - test { - sql """ - CREATE TABLE test_complex_default_value_bad_json ( - k INT, - v JSON DEFAULT '{}' - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES('replication_num'='1') - """ - exception "only supports DEFAULT NULL" + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"bad": 1}'""" + exception "Invalid default value" } - test { - sql """ - CREATE TABLE test_complex_default_value_bad_variant ( - k INT, - v VARIANT DEFAULT '{}' - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES('replication_num'='1') - """ - exception "only supports DEFAULT NULL" - } - - // INSERT fills unmentioned columns through a FE-side cast of the default literal, which is - // non-strict by default, so BE default parsing is exercised by reading rows that were written - // before the column with the invalid default was added by light schema change. - def checkInvalidDefaultOnRead = { String tableName, String columnDef -> - sql """ - CREATE TABLE ${tableName} ( - k INT - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES( - 'replication_num'='1', - 'light_schema_change'='true' - ) - """ - sql "INSERT INTO ${tableName} VALUES (1)" - sql "ALTER TABLE ${tableName} ADD COLUMN v ${columnDef}" - waitForSchemaChangeDone { - sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1""" - time 600 - } - test { - sql "SELECT k, v FROM ${tableName}" - exception "parse number fail" - } - } - - checkInvalidDefaultOnRead.call("test_complex_default_value_bad_array_elem", - """ARRAY DEFAULT '["bad"]'""") - checkInvalidDefaultOnRead.call("test_complex_default_value_bad_map_key", - """MAP DEFAULT '{"bad": 1}'""") - checkInvalidDefaultOnRead.call("test_complex_default_value_bad_map_value", - """MAP DEFAULT '{"bad": "value"}'""") - checkInvalidDefaultOnRead.call("test_complex_default_value_bad_struct_field", - """STRUCT DEFAULT '{"bad"}'""") - - // Cloud mode may finish add-column schema change as a metadata-only operation, while local - // storage rewrites existing rows and cancels when the invalid default is materialized. - if (!isCloudMode()) { - sql """ - CREATE TABLE test_complex_default_value_bad_alter_array_elem ( - k INT - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES( - 'replication_num'='1', - 'light_schema_change'='false' - ) - """ - - sql "INSERT INTO test_complex_default_value_bad_alter_array_elem VALUES (1)" - sql """ - ALTER TABLE test_complex_default_value_bad_alter_array_elem - ADD COLUMN v ARRAY DEFAULT '["bad"]' - """ - waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_array_elem") - - sql """ - CREATE TABLE test_complex_default_value_bad_alter_map_key ( - k INT - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES( - 'replication_num'='1', - 'light_schema_change'='false' - ) - """ - - sql "INSERT INTO test_complex_default_value_bad_alter_map_key VALUES (1)" - sql """ - ALTER TABLE test_complex_default_value_bad_alter_map_key - ADD COLUMN v MAP DEFAULT '{"bad": 1}' - """ - waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_map_key") - - sql """ - CREATE TABLE test_complex_default_value_bad_alter_struct_field ( - k INT - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES( - 'replication_num'='1', - 'light_schema_change'='false' - ) - """ - - sql "INSERT INTO test_complex_default_value_bad_alter_struct_field VALUES (1)" - sql """ - ALTER TABLE test_complex_default_value_bad_alter_struct_field - ADD COLUMN v STRUCT DEFAULT '{"bad"}' - """ - waitForCancelledSchemaChange.call("test_complex_default_value_bad_alter_struct_field") + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v STRUCT DEFAULT '{"bad"}'""" + exception "Invalid default value" } sql """ @@ -205,7 +95,14 @@ suite("test_complex_default_value") { struct_literal STRUCT DEFAULT '{7, "x"}', arr_nested_null ARRAY DEFAULT '[NULL, nUlL, 5]', map_nested_null MAP DEFAULT '{"upper": NULL, "mixed": nUlL, "value": 6}', - struct_nested_null STRUCT DEFAULT '{NULL, NuLl, 7}' + struct_nested_null STRUCT DEFAULT '{NULL, NuLl, 7}', + arr_typed_date ARRAY DEFAULT '[DATEV2 "2024-01-01", "2024-02-02"]', + arr_exponent ARRAY DEFAULT '[1e3, "7"]', + arr_bool ARRAY DEFAULT '[true, false]', + arr_decimal ARRAY DEFAULT '[1.234]', + arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', + map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + arr_nested ARRAY> DEFAULT '[[1], [], NULL]' ) DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 @@ -214,10 +111,24 @@ suite("test_complex_default_value") { sql "INSERT INTO test_complex_default_value_literal(k) VALUES (1)" order_qt_literal_default """ - SELECT k, arr_empty, arr_literal, map_empty, map_literal, struct_empty, struct_literal, - arr_nested_null, map_nested_null, struct_nested_null - FROM test_complex_default_value_literal - ORDER BY k + SELECT * FROM test_complex_default_value_literal ORDER BY k + """ + + // SHOW CREATE TABLE renders the canonical default in a form that CREATE TABLE LIKE can replay + def showCreate = sql "SHOW CREATE TABLE test_complex_default_value_literal" + def createSql = showCreate[0][1] + logger.info("show create table: ${createSql}") + assertTrue(createSql.contains('`arr_literal` array NULL DEFAULT "[1, 2]"')) + assertTrue(createSql.contains('`map_literal` map NULL DEFAULT \'{"a":10, "b":20}\'')) + assertTrue(createSql.contains('`arr_typed_date` array NULL DEFAULT \'["2024-01-01", "2024-02-02"]\'')) + assertTrue(createSql.contains('`arr_exponent` array NULL DEFAULT "[1000, 7]"')) + assertTrue(createSql.contains('`arr_bool` array NULL DEFAULT "[1, 0]"')) + assertTrue(createSql.contains('`map_repeated_key` map NULL DEFAULT \'{"a":2}\'')) + assertTrue(createSql.contains('`arr_string` array NULL DEFAULT \'["x,y", "[z]", "{k:v}", "null", ""]\'')) + sql "CREATE TABLE test_complex_default_value_like LIKE test_complex_default_value_literal" + sql "INSERT INTO test_complex_default_value_like(k) VALUES (1)" + order_qt_like_default """ + SELECT * FROM test_complex_default_value_like ORDER BY k """ sql """ @@ -241,6 +152,7 @@ suite("test_complex_default_value") { ORDER BY k """ + // rows written before the columns were added read the defaults through the BE default value iterator sql """ CREATE TABLE test_complex_default_value_alter_base ( k INT @@ -258,7 +170,14 @@ suite("test_complex_default_value") { ALTER TABLE test_complex_default_value_alter_base ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', ADD COLUMN map_added MAP NOT NULL DEFAULT '{"z": 9}', - ADD COLUMN struct_added STRUCT NOT NULL DEFAULT '{8, "y"}' + ADD COLUMN struct_added STRUCT NOT NULL DEFAULT '{8, "y"}', + ADD COLUMN arr_typed_date ARRAY DEFAULT '[DATEV2 "2024-01-01", "2024-02-02"]', + ADD COLUMN arr_exponent ARRAY DEFAULT '[1e3, "7"]', + ADD COLUMN arr_bool ARRAY DEFAULT '[true, false]', + ADD COLUMN arr_decimal ARRAY DEFAULT '[1.234]', + ADD COLUMN arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', + ADD COLUMN map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + ADD COLUMN arr_nested ARRAY> DEFAULT '[[1], [], NULL]' """ waitForSchemaChangeDone { sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_base' ORDER BY createtime DESC LIMIT 1""" @@ -266,15 +185,61 @@ suite("test_complex_default_value") { } order_qt_alter_default """ - SELECT k, arr_added, map_added, struct_added - FROM test_complex_default_value_alter_base - ORDER BY k + SELECT * FROM test_complex_default_value_alter_base ORDER BY k """ sql "INSERT INTO test_complex_default_value_alter_base(k) VALUES (2)" order_qt_alter_default_after_insert """ - SELECT k, arr_added, map_added, struct_added - FROM test_complex_default_value_alter_base - ORDER BY k + SELECT * FROM test_complex_default_value_alter_base ORDER BY k + """ + + // direct schema change materializes the defaults into the rewritten rows + sql """ + CREATE TABLE test_complex_default_value_alter_direct ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + 'replication_num'='1', + 'light_schema_change'='false' + ) + """ + + sql "INSERT INTO test_complex_default_value_alter_direct VALUES (1)" + sql """ + ALTER TABLE test_complex_default_value_alter_direct + ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', + ADD COLUMN map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', + ADD COLUMN struct_added STRUCT> DEFAULT '{8, [DATEV2 "2024-01-01"]}', + ADD COLUMN arr_string ARRAY DEFAULT '["x,y", ""]' + """ + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_direct' ORDER BY createtime DESC LIMIT 1""" + time 600 + } + + order_qt_alter_direct_default """ + SELECT * FROM test_complex_default_value_alter_direct ORDER BY k + """ + + // a one-argument replace_value mapping falls back to the complex column default + sql """ + CREATE TABLE test_complex_default_value_replace_value ( + k INT, + v MAP DEFAULT '{"a": 1}' + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + streamLoad { + table "test_complex_default_value_replace_value" + set 'columns', 'k, v, v = replace_value(null)' + file "test_complex_default_value_replace_value.csv" + time 60 + } + order_qt_replace_value """ + SELECT k, v FROM test_complex_default_value_replace_value ORDER BY k """ } From f5f476babe22128c0f351b7c2983144a76f2ceb7 Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Fri, 9 Oct 2026 15:30:28 +0800 Subject: [PATCH 3/6] [fix](complex) Canonicalize explicit load mappings and reject colliding map keys Issue Number: close #xxx Related PR: #xxx Problem Summary: Follow-up fixes for literal defaults of ARRAY/MAP/STRUCT columns. 1. `default_value('[1e3]')` and `replace_value(null, '[1e3]')` mappings on an ARRAY column passed FE validation (which canonicalizes the text to `[1000]`) but the original literal was still sent to BE, whose string cast writes a NULL element in non-strict load and fails in strict load. `NereidsLoadScanProvider.transformHadoopFunctionExpr` now replaces the validated argument with the canonical text. 2. `MAP DEFAULT '{"01": 1, "1": 2}'` rendered two entries with key `1`; BE materialized both for old rows while replaying the text as a SQL literal kept only the last one. Keys that collide after the cast to the key type are rejected at DDL time. 3. `Segment::new_default_iterator` built a (recursive) SerDe for every missing column of every old segment even when the column has no default or `DEFAULT NULL`, which never parses anything. The SerDe is now built only for a non-NULL default text and `DefaultValueColumnIterator` asserts it exists when it must parse; the iterator's unused FieldType/precision/scale/len fields are removed. 4. The regression case labelled as direct schema change actually took BE's linked strategy; a delete predicate is now added before the ALTER so BE selects `sc_directly` and materializes the complex defaults. None - Test: Unit Test / Regression test - FE UT: ColumnDefinitionTest, NereidsLoadScanProviderTest - BE UT: ColumnReaderTest, DataTypeSerDeFromStringTest - Regression: datatype_p0/complex_types/test_complex_default_value, schema_change_p0 (test_dup_schema_key_add, test_modify_struct, test_unique_schema_key_change_add) - Behavior changed: Yes (map defaults whose keys collide after the cast are rejected; explicit complex load mapping values are canonicalized) - Does this need documentation: No --- .../nereids/load/NereidsLoadScanProvider.java | 22 ++++- .../info/ComplexTypeDefaultValue.java | 15 +++- .../load/NereidsLoadScanProviderTest.java | 86 ++++++++++++++++++- .../commands/info/ColumnDefinitionTest.java | 5 ++ .../test_complex_default_value.out | 10 +++ .../test_complex_default_value_mapping.csv | 2 + .../test_complex_default_value.groovy | 47 ++++++++++ 7 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java index 8b22fca5409026..633c4e874931c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java @@ -43,7 +43,9 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.commands.info.ComplexTypeDefaultValue; import org.apache.doris.nereids.trees.plans.commands.info.DefaultValue; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VarcharType; @@ -483,7 +485,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp exprs.add(new Not(new IsNull(slot))); exprs.add(slot); if (funcExpr.children().size() > 1) { - exprs.add(funcExpr.child(1)); + exprs.add(canonicalizeComplexValue(column, funcExpr.child(1))); } else { if (column.getDefaultValue() != null) { String exprSql = column.getDefaultValueSql(); @@ -503,7 +505,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp innerIfExprs.add(new Not(new EqualTo(slot, funcExpr.child(0)))); innerIfExprs.add(slot); if (funcExpr.children().size() > 1) { - innerIfExprs.add(funcExpr.child(1)); + innerIfExprs.add(canonicalizeComplexValue(column, funcExpr.child(1))); } else { if (column.getDefaultValue() != null) { String exprSql = column.getDefaultValueSql(); @@ -571,7 +573,7 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp return unixTimeFunc; } else if (funcName.equalsIgnoreCase("default_value")) { - return funcExpr.child(0); + return canonicalizeComplexValue(column, funcExpr.child(0)); } else if (funcName.equalsIgnoreCase("now")) { UnboundFunction newFunc = new UnboundFunction("now", Lists.newArrayList()); return newFunc; @@ -581,4 +583,18 @@ private Expression transformHadoopFunctionExpr(Table tbl, String columnName, Exp } return originExpr; } + + /** + * The explicit value of a default_value or replace_value mapping on an ARRAY/MAP/STRUCT column was + * validated by {@link NereidsDataDescription#validateMappingFunction} as a SQL literal, but BE casts + * the mapping expression as a string. Replace it with the canonical text so that both agree, e.g. + * {@code '[1e3]'} for an array of INT is written as {@code [1000]} instead of a NULL element. + */ + private static Expression canonicalizeComplexValue(Column column, Expression value) { + if (!column.getOriginType().isComplexType() || !(value instanceof StringLikeLiteral)) { + return value; + } + return new StringLiteral(ComplexTypeDefaultValue.canonicalize( + DataType.fromCatalogType(column.getOriginType()), ((StringLikeLiteral) value).getStringValue())); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java index 421af8ad72c638..4cb28c5a101848 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java @@ -34,8 +34,10 @@ import com.google.common.base.Preconditions; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.StringJoiner; /** @@ -120,11 +122,18 @@ private static String render(Literal literal, DataType type) throws AnalysisExce throw new AnalysisException(literal.toSql() + " is not a map literal"); } MapType mapType = (MapType) type; - // The parser keeps the last value of a repeated key, so a repeated key is stored once. + // The parser keeps the last value of a repeated key, so a repeated key is stored once. Keys that + // only collide after the cast to the key type (e.g. "01" and "1" for INT) are rejected: BE keeps + // every entry of the stored text, while replaying it as a SQL literal would keep only the last. + Set keys = new HashSet<>(); StringJoiner joiner = new StringJoiner(", ", "{", "}"); for (Map.Entry entry : ((MapLiteral) literal).getValue().entrySet()) { - joiner.add(render(entry.getKey(), mapType.getKeyType()) + ":" - + render(entry.getValue(), mapType.getValueType())); + String key = render(entry.getKey(), mapType.getKeyType()); + if (!keys.add(key)) { + throw new AnalysisException(String.format("map key %s is repeated after casting to %s", key, + mapType.getKeyType().toSql())); + } + joiner.add(key + ":" + render(entry.getValue(), mapType.getValueType())); } return joiner.toString(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java index 8a785370a9fff3..a2c8c5d06d7246 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsLoadScanProviderTest.java @@ -18,16 +18,22 @@ package org.apache.doris.nereids.load; import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.fileformat.ArrowFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; @@ -152,6 +158,63 @@ public void testImportColumnMatchesTableColumnCaseInsensitive() { ImmutableList.of(new NereidsImportColumnDesc("other")), tableColumn)); } + @Test + public void testComplexColumnMappingValueIsCanonicalized() throws Exception { + OlapTable table = mockComplexTable(); + StringLiteral canonical = new StringLiteral("[1000, 7]"); + + // v = default_value('[1e3, "7"]') + NereidsParamCreateContext context = createMappingLoadContext(table, + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("[1e3, \"7\"]"))), + Pair.of("default_value", Lists.newArrayList("[1e3, \"7\"]"))); + Assertions.assertEquals(canonical, context.exprMap.get("v")); + + // v = replace_value(null, '[1e3, "7"]') becomes if(v is not null, v, '[1000, 7]') + context = createMappingLoadContext(table, + new UnboundFunction("replace_value", ImmutableList.of(new NullLiteral(), + new StringLiteral("[1e3, \"7\"]"))), + Pair.of("replace_value", Lists.newArrayList(null, "[1e3, \"7\"]"))); + Expression replaced = context.exprMap.get("v"); + Assertions.assertTrue(replaced instanceof UnboundFunction, replaced.toSql()); + Assertions.assertEquals(canonical, replaced.child(2)); + + // v = replace_value('[]', '[1e3, "7"]') becomes if(v is not null, if(v != '[]', v, '[1000, 7]'), null) + context = createMappingLoadContext(table, + new UnboundFunction("replace_value", ImmutableList.of(new StringLiteral("[]"), + new StringLiteral("[1e3, \"7\"]"))), + Pair.of("replace_value", Lists.newArrayList("[]", "[1e3, \"7\"]"))); + replaced = context.exprMap.get("v"); + Assertions.assertTrue(replaced instanceof UnboundFunction, replaced.toSql()); + Assertions.assertEquals(canonical, replaced.child(1).child(2)); + + // the value is validated as a literal of the column type before it is rewritten + UserException exception = Assertions.assertThrows(UserException.class, () -> createMappingLoadContext(table, + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("[\"bad\"]"))), + Pair.of("default_value", Lists.newArrayList("[\"bad\"]")))); + Assertions.assertTrue(exception.getMessage().contains("Invalid default value"), exception.getMessage()); + + // a scalar column keeps its mapping value verbatim + context = createMappingLoadContext(table, "k", + new UnboundFunction("default_value", ImmutableList.of(new StringLiteral("007"))), + Pair.of("default_value", Lists.newArrayList("007"))); + Assertions.assertEquals(new StringLiteral("007"), context.exprMap.get("k")); + } + + private NereidsParamCreateContext createMappingLoadContext(OlapTable table, Expression mapping, + Pair> hadoopFunction) throws UserException { + return createMappingLoadContext(table, "v", mapping, hadoopFunction); + } + + private NereidsParamCreateContext createMappingLoadContext(OlapTable table, String column, Expression mapping, + Pair> hadoopFunction) throws UserException { + Map>> columnToHadoopFunction = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + columnToHadoopFunction.put(column, hadoopFunction); + return createLoadContext(table, + ImmutableList.of(new NereidsImportColumnDesc("k"), new NereidsImportColumnDesc("v"), + new NereidsImportColumnDesc(column, mapping)), + columnToHadoopFunction, new ArrowFileFormatProperties()); + } + private NereidsParamCreateContext createArrowLoadContext( OlapTable table, List columnExprList) throws UserException { return createLoadContext(table, columnExprList, new ArrowFileFormatProperties()); @@ -160,9 +223,15 @@ private NereidsParamCreateContext createArrowLoadContext( private NereidsParamCreateContext createLoadContext(OlapTable table, List columnExprList, FileFormatProperties fileFormatProperties) throws UserException { + return createLoadContext(table, columnExprList, null, fileFormatProperties); + } + + private NereidsParamCreateContext createLoadContext(OlapTable table, List columnExprList, + Map>> columnToHadoopFunction, FileFormatProperties fileFormatProperties) + throws UserException { NereidsBrokerFileGroup fileGroup = new NereidsBrokerFileGroup(1L, false, null, - Lists.newArrayList("dummy"), null, null, null, columnExprList, null, null, null, null, - LoadTask.MergeType.APPEND, null, -1L, false, false, fileFormatProperties); + Lists.newArrayList("dummy"), null, null, null, columnExprList, columnToHadoopFunction, null, null, + null, LoadTask.MergeType.APPEND, null, -1L, false, false, fileFormatProperties); TBrokerFileStatus fileStatus = new TBrokerFileStatus(); fileStatus.setPath(""); fileStatus.setIsDir(false); @@ -174,10 +243,13 @@ private NereidsParamCreateContext createLoadContext(OlapTable table, List schema = Arrays.asList( + return mockTable(Arrays.asList( new Column("time", PrimitiveType.DATETIME, true), new Column("securityid", PrimitiveType.INT, true), - new Column("EV", PrimitiveType.DOUBLE, true)); + new Column("EV", PrimitiveType.DOUBLE, true))); + } + + private OlapTable mockTable(List schema) { Map nameToColumn = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (Column column : schema) { nameToColumn.put(column.getName(), column); @@ -198,6 +270,12 @@ private OlapTable mockTable() { return table; } + private OlapTable mockComplexTable() { + return mockTable(Arrays.asList( + new Column("k", PrimitiveType.INT, true), + new Column("v", ArrayType.create(ScalarType.INT, true), true))); + } + private void assertSlot(NereidsParamCreateContext context, String name, PrimitiveType type) { SlotReference slot = context.scanSlots.stream() .filter(scanSlot -> scanSlot.getName().equals(name)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index 575f1a1bc1067a..dcf17787aed112 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -200,6 +200,11 @@ public void testComplexTypeLiteralDefaultValue() { assertRejectsDefaultValue(stringIntMap, "{\"a\": \"bad\"}", "Invalid default value"); assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"bad\": 1}", "Invalid default value"); + // keys that are distinct literals but collide after the cast to the key type are rejected + assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"01\": 1, \"1\": 2}", + "map key 1 is repeated after casting to INT"); + assertCanonicalDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"01\": 1, 2: 2}", + "{1:1, 2:2}"); StructType structType = new StructType(Arrays.asList( new StructField("f1", IntegerType.INSTANCE, true, ""), diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out index 27808c9f67f77b..bd638b931e7e2d 100644 --- a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out @@ -22,3 +22,13 @@ 1 {"a":1} 2 {"b":2} +-- !mapping_replace_value -- +1 [1000, 7] +2 [5] + +-- !mapping_default_value -- +1 [1000, 7] +1 [1000, 7] +2 [1000, 7] +2 [5] + diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv new file mode 100644 index 00000000000000..2e4e963e54f98a --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value_mapping.csv @@ -0,0 +1,2 @@ +1 \N +2 [5] diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy index e33f39e4ab04b9..254e99c2c1fa53 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -23,6 +23,7 @@ suite("test_complex_default_value") { sql "DROP TABLE IF EXISTS test_complex_default_value_alter_direct" sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter" sql "DROP TABLE IF EXISTS test_complex_default_value_replace_value" + sql "DROP TABLE IF EXISTS test_complex_default_value_mapping" def createTableRejects = { String columnDef, String message -> test { @@ -55,6 +56,8 @@ suite("test_complex_default_value") { createTableRejects.call("""STRUCT DEFAULT '{"bad"}'""", "Invalid default value") createTableRejects.call("""STRUCT DEFAULT '{1}'""", "struct literal has 1 fields but the column has 2") createTableRejects.call("""ARRAY> DEFAULT '[[1], ["bad"]]'""", "Invalid default value") + // map keys that only collide after the cast to the key type are rejected + createTableRejects.call("""MAP DEFAULT '{"01": 1, "1": 2}'""", "map key 1 is repeated after casting to INT") // nested string values are stored as plain double quoted text, so quotes and backslashes are rejected createTableRejects.call("""ARRAY DEFAULT '["a""b"]'""", "must not contain quote or backslash") @@ -83,6 +86,10 @@ suite("test_complex_default_value") { sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v STRUCT DEFAULT '{"bad"}'""" exception "Invalid default value" } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"01": 1, "1": 2}'""" + exception "map key 1 is repeated after casting to INT" + } sql """ CREATE TABLE test_complex_default_value_literal ( @@ -207,6 +214,9 @@ suite("test_complex_default_value") { """ sql "INSERT INTO test_complex_default_value_alter_direct VALUES (1)" + // a delete predicate on the base rowsets makes BE pick the direct strategy instead of linking + // the old rowset, so the new columns are materialized by the schema change itself + sql "DELETE FROM test_complex_default_value_alter_direct WHERE k = 0" sql """ ALTER TABLE test_complex_default_value_alter_direct ADD COLUMN arr_added ARRAY NOT NULL DEFAULT '[3, 4]', @@ -242,4 +252,41 @@ suite("test_complex_default_value") { order_qt_replace_value """ SELECT k, v FROM test_complex_default_value_replace_value ORDER BY k """ + + // an explicit default_value / replace_value argument of a complex column is written as the + // canonical literal, so 1e3 and "7" become INT elements instead of NULL + sql """ + CREATE TABLE test_complex_default_value_mapping ( + k INT, + v ARRAY + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + streamLoad { + table "test_complex_default_value_mapping" + set 'columns', 'k, v, v = replace_value(null, \'[1e3, "7"]\')' + file "test_complex_default_value_mapping.csv" + time 60 + } + order_qt_mapping_replace_value """ + SELECT k, v FROM test_complex_default_value_mapping ORDER BY k + """ + streamLoad { + table "test_complex_default_value_mapping" + set 'columns', 'k, tmp, v = default_value(\'[1e3, "7"]\')' + file "test_complex_default_value_mapping.csv" + time 60 + } + order_qt_mapping_default_value """ + SELECT k, v FROM test_complex_default_value_mapping ORDER BY k, v + """ + test { + sql """ + ALTER TABLE test_complex_default_value_mapping + ADD COLUMN bad MAP DEFAULT '{"01": 1, "1": 2}' + """ + exception "map key 1 is repeated after casting to INT" + } } From 9908012c3e4977b656fe58b17ab512439b0203d8 Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Fri, 9 Oct 2026 16:29:17 +0800 Subject: [PATCH 4/6] [fix](complex) Parse complex default literals without item type coercion ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: Follow-up fix for literal defaults of ARRAY/MAP/STRUCT columns. `ComplexTypeDefaultValue.canonicalize` parsed the default text with the regular SQL literal builder, which first casts the items of an array literal and the keys and values of a map literal to a common type inferred from the items. With the default `enable_strict_cast=false` an item that cannot be cast is silently replaced by NULL, so `ARRAY DEFAULT '[DATEV2 "2024-01-01", 1]'` was accepted and stored as `["2024-01-01", NULL]`: old rows and INSERTs that omit the column got a NULL the user never wrote. The default text is now parsed with a builder that keeps every array item and map key/value exactly as written; `render` casts each of them to the declared nested type with `checkedCastTo`, which rejects the invalid item at DDL time regardless of the session setting. `NereidsParser.parseExpression` gains an overload taking the builder, mirroring `parseSingle`. ### Release note None ### Check List (For Author) - Test: Unit Test / Regression test - FE UT: ColumnDefinitionTest, NereidsLoadScanProviderTest, AlterTableCommandTest - Regression: datatype_p0/complex_types/test_complex_default_value - Behavior changed: Yes (a nested default value that the SQL parser would have coerced to NULL is now rejected) - Does this need documentation: No --- .../doris/nereids/parser/NereidsParser.java | 13 +++- .../info/ComplexTypeDefaultValue.java | 60 ++++++++++++++++++- .../commands/info/ColumnDefinitionTest.java | 17 ++++++ .../test_complex_default_value.groovy | 14 +++++ 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 7a30b276da158f..1ea4a4986f1305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -304,10 +304,21 @@ public List> parseMultiple(String sql, } public Expression parseExpression(String expression) { + return parseExpression(expression, null); + } + + /** + * parse an expression with the given logical plan builder. + * + * @param expression expression string + * @param logicalPlanBuilder logical plan builder, null for the default one + * @return expression + */ + public Expression parseExpression(String expression, @Nullable LogicalPlanBuilder logicalPlanBuilder) { if (isValidUnquotedIdentifier(expression)) { return new UnboundSlot(expression); } - return parse(expression, DorisParser::expressionWithEof); + return parse(expression, logicalPlanBuilder, DorisParser::expressionWithEof); } /** Return whether the text can be emitted as an unquoted Nereids identifier. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java index 4cb28c5a101848..f0e0c8e98d1fe1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java @@ -17,7 +17,12 @@ package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.nereids.DorisParser.ArrayLiteralContext; +import org.apache.doris.nereids.DorisParser.ConstantContext; +import org.apache.doris.nereids.DorisParser.MapLiteralContext; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.LogicalPlanBuilder; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; @@ -33,8 +38,12 @@ import org.apache.doris.nereids.types.StructType; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.ParserRuleContext; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,8 +52,9 @@ /** * Validates a non-null ARRAY/MAP/STRUCT column default and rewrites it into a canonical literal text. * - *

The user supplied text is parsed as a SQL literal and every nested value is cast to the declared - * nested type, so a type mismatch is rejected at DDL time instead of when old rows are read. The + *

The user supplied text is parsed as a SQL literal whose nested values are kept exactly as written + * (see {@link LiteralBuilder}), and every nested value is then cast to the declared nested type, so a + * type mismatch is rejected at DDL time instead of when old rows are read. The * canonical text only contains plain nested values (unquoted numbers, double quoted strings, nested * brackets and NULL). BE parses the stored text in two places that must agree: the complex SerDe * {@code from_fe_string} used by the default value iterator and schema change, and the @@ -65,7 +75,7 @@ public static String canonicalize(DataType type, String defaultValue) throws Ana "%s is not a complex type", type); Expression expression; try { - expression = new NereidsParser().parseExpression(defaultValue); + expression = new NereidsParser().parseExpression(defaultValue, new LiteralBuilder()); } catch (Exception e) { throw literalShapeException(type); } @@ -181,4 +191,48 @@ private static String renderScalar(Literal literal, DataType type) throws Analys } throw new AnalysisException(type.toSql() + " is not supported as a nested type of a default value"); } + + /** + * Builds array and map literals from their items as written. + * + *

{@link LogicalPlanBuilder} first casts the items of an array literal, and the keys and the values of a + * map literal, to a common type inferred from the items, and with {@code enable_strict_cast=false} an item + * that cannot be cast silently becomes NULL: {@code '[DATEV2 "2024-01-01", 1]'} would be stored as + * {@code ["2024-01-01", NULL]}. The items of a default value are cast to the declared nested type by + * {@link #render} instead, which rejects such an item, so no coercion happens while parsing. + */ + private static final class LiteralBuilder extends LogicalPlanBuilder { + private LiteralBuilder() { + super(ImmutableMap.of()); + } + + @Override + public ArrayLiteral visitArrayLiteral(ArrayLiteralContext ctx) { + return new ArrayLiteral(visitItems(ctx, ctx.items)); + } + + @Override + public MapLiteral visitMapLiteral(MapLiteralContext ctx) { + // the grammar only accepts complete key:value pairs + List items = visitItems(ctx, ctx.items); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < items.size(); i += 2) { + map.put(items.get(i), items.get(i + 1)); + } + return new MapLiteral(map); + } + + private List visitItems(ParserRuleContext ctx, List items) { + ImmutableList.Builder literals = ImmutableList.builderWithExpectedSize(items.size()); + for (ConstantContext item : items) { + Expression expression = typedVisit(item); + // a placeholder is the only constant that is not a literal + if (!(expression instanceof Literal)) { + throw new ParseException("default value must only contain literals", ctx); + } + literals.add((Literal) expression); + } + return literals.build(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index dcf17787aed112..5c2e4100eff948 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.UuidType; import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -179,6 +180,15 @@ public void testComplexTypeLiteralDefaultValue() { assertCanonicalDefaultValue(ArrayType.of(DateV2Type.INSTANCE), "[DATEV2 \"2024-01-01\", \"2024-02-02\"]", "[\"2024-01-01\", \"2024-02-02\"]"); + // an item that cannot be cast to the type inferred from the other items is rejected instead of + // being replaced by NULL, which the SQL literal parser does when enable_strict_cast is off + Assertions.assertFalse(SessionVariable.enableStrictCast()); + assertRejectsDefaultValue(ArrayType.of(DateV2Type.INSTANCE), "[DATEV2 \"2024-01-01\", 1]", + "Invalid default value '[DATEV2 \"2024-01-01\", 1]' for ARRAY column"); + // the parser would infer DATEV2 for the items and turn 1 into NULL; each item is cast to INT instead + assertCanonicalDefaultValue(intArray, "[DATEV2 \"2024-01-01\", 1]", "[20240101, 1]"); + assertRejectsDefaultValue(ArrayType.of(ArrayType.of(DateV2Type.INSTANCE)), + "[[DATEV2 \"2024-01-01\"], [1]]", "Invalid default value"); assertCanonicalDefaultValue(ArrayType.of(BooleanType.INSTANCE), "[true, false]", "[1, 0]"); assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[1.5, 2]", "[1.5, 2.0]"); assertCanonicalDefaultValue(ArrayType.of(DecimalV3Type.createDecimalV3Type(10, 2)), "[1.234]", "[1.23]"); @@ -200,6 +210,10 @@ public void testComplexTypeLiteralDefaultValue() { assertRejectsDefaultValue(stringIntMap, "{\"a\": \"bad\"}", "Invalid default value"); assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"bad\": 1}", "Invalid default value"); + assertRejectsDefaultValue(MapType.of(DateV2Type.INSTANCE, IntegerType.INSTANCE), + "{DATEV2 \"2024-01-01\": 1, 1: 2}", "Invalid default value"); + assertRejectsDefaultValue(MapType.of(StringType.INSTANCE, DateV2Type.INSTANCE), + "{\"a\": DATEV2 \"2024-01-01\", \"b\": 1}", "Invalid default value"); // keys that are distinct literals but collide after the cast to the key type are rejected assertRejectsDefaultValue(MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE), "{\"01\": 1, \"1\": 2}", "map key 1 is repeated after casting to INT"); @@ -216,6 +230,9 @@ public void testComplexTypeLiteralDefaultValue() { "only supports struct literals or DEFAULT NULL"); assertRejectsDefaultValue(structType, "{1}", "struct literal has 1 fields but the column has 2"); assertRejectsDefaultValue(structType, "{\"bad\", \"a\"}", "Invalid default value"); + assertRejectsDefaultValue(new StructType(Arrays.asList( + new StructField("f1", ArrayType.of(DateV2Type.INSTANCE), true, ""))), + "{[DATEV2 \"2024-01-01\", 1]}", "Invalid default value"); assertRejectsDefaultValue(structType, "[]", "only supports struct literals or DEFAULT NULL"); assertRejectsDefaultValue(JsonType.INSTANCE, "{}", "only supports DEFAULT NULL"); diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy index 254e99c2c1fa53..8dffa1b6919f03 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -56,6 +56,12 @@ suite("test_complex_default_value") { createTableRejects.call("""STRUCT DEFAULT '{"bad"}'""", "Invalid default value") createTableRejects.call("""STRUCT DEFAULT '{1}'""", "struct literal has 1 fields but the column has 2") createTableRejects.call("""ARRAY> DEFAULT '[[1], ["bad"]]'""", "Invalid default value") + // an item that cannot be cast to the type inferred from the other items is rejected instead of being + // replaced by NULL, which the SQL literal parser does under the default enable_strict_cast=false + createTableRejects.call("""ARRAY DEFAULT '[DATEV2 "2024-01-01", 1]'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{DATEV2 "2024-01-01": 1, 1: 2}'""", "Invalid default value") + createTableRejects.call("""MAP DEFAULT '{"a": DATEV2 "2024-01-01", "b": 1}'""", "Invalid default value") + createTableRejects.call("""STRUCT> DEFAULT '{[DATEV2 "2024-01-01", 1]}'""", "Invalid default value") // map keys that only collide after the cast to the key type are rejected createTableRejects.call("""MAP DEFAULT '{"01": 1, "1": 2}'""", "map key 1 is repeated after casting to INT") @@ -90,6 +96,14 @@ suite("test_complex_default_value") { sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"01": 1, "1": 2}'""" exception "map key 1 is repeated after casting to INT" } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '[DATEV2 "2024-01-01", 1]'""" + exception "Invalid default value" + } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"a": DATEV2 "2024-01-01", "b": 1}'""" + exception "Invalid default value" + } sql """ CREATE TABLE test_complex_default_value_literal ( From 56292050219133897db92ad055709554f3e16736 Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Tue, 13 Oct 2026 18:05:55 +0800 Subject: [PATCH 5/6] [test](be) Cover non-empty complex literal defaults in constant-column test ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Rebasing onto master picked up #65684, which deleted `DefaultValueColumnIterator` and moved default-value parsing into `Segment::get_default_value_field`, together with a new `constant_column_iterator_test.cpp`. That file's `NonEmptyComplexTextDefaultsAreRejected` asserted the pre-PR restriction that a non-empty ARRAY/MAP/STRUCT literal default is rejected (`NOT_IMPLEMENTED_ERROR`), which no longer holds once this branch's earlier commits teach the complex SerDes to parse such literals. Replace it with `NonEmptyComplexTextDefaultsAreParsed`, asserting that `[1]`, `{"k":7}`, and a positional struct literal are parsed into the matching typed `Field` through `Segment::get_default_value_field`. ### Release note None ### Check List (For Author) - Test: Unit Test - `./run-be-ut.sh --run --filter="ConstantColumnIteratorTest.*" -j 32` (34 tests, all pass) - Behavior changed: No (test-only; the behavior change itself is in the earlier commits) - Does this need documentation: No --- .../segment/constant_column_iterator_test.cpp | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/be/test/storage/segment/constant_column_iterator_test.cpp b/be/test/storage/segment/constant_column_iterator_test.cpp index 27e5ef383ec944..094c6eea767a2b 100644 --- a/be/test/storage/segment/constant_column_iterator_test.cpp +++ b/be/test/storage/segment/constant_column_iterator_test.cpp @@ -973,9 +973,10 @@ TEST_F(ConstantColumnIteratorTest, StructNullDefaultAndRuntimeConstantMaterializ EXPECT_EQ("doris", materialized_struct->get_column(1).get_data_at(1).to_string()); } -TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { - // Case: schema defaults currently support only [] and {} for complex non-null values. Reject - // non-empty text forms explicitly instead of silently synthesizing a different value. +TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreParsed) { + // Case: a non-empty ARRAY/MAP/STRUCT literal default is parsed through the declared (nested) + // type's SerDe instead of being rejected, so `[1]`, `{"k":7}`, and a positional struct literal + // all materialize the matching typed Field. TabletColumn int_child; int_child.set_type(FieldType::OLAP_FIELD_TYPE_INT); int_child.set_is_nullable(false); @@ -988,9 +989,11 @@ TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { array_column.set_is_nullable(false); array_column.set_default_value("[1]"); array_column.add_sub_column(int_child); - Field field; - EXPECT_TRUE(Segment::get_default_value_field(array_column, &field) - .is()); + Field array_field; + ASSERT_TRUE(Segment::get_default_value_field(array_column, &array_field).ok()); + const auto& array = array_field.get(); + ASSERT_EQ(1, array.size()); + EXPECT_EQ(1, array[0].get()); TabletColumn map_column; map_column.set_type(FieldType::OLAP_FIELD_TYPE_MAP); @@ -998,17 +1001,31 @@ TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreRejected) { map_column.set_default_value("{\"k\":7}"); map_column.add_sub_column(string_child); map_column.add_sub_column(int_child); - EXPECT_TRUE(Segment::get_default_value_field(map_column, &field) - .is()); - + Field map_field; + ASSERT_TRUE(Segment::get_default_value_field(map_column, &map_field).ok()); + const auto& map = map_field.get(); + ASSERT_EQ(2, map.size()); + const auto& keys = map[0].get(); + const auto& values = map[1].get(); + ASSERT_EQ(1, keys.size()); + ASSERT_EQ(1, values.size()); + EXPECT_EQ("k", std::string_view(keys[0].get())); + EXPECT_EQ(7, values[0].get()); + + // Positional struct literal: int_child/string_child have no declared name, so the literal is + // matched by field position instead of by name. TabletColumn struct_column; struct_column.set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); struct_column.set_is_nullable(false); - struct_column.set_default_value("(99, doris)"); + struct_column.set_default_value("{99,\"doris\"}"); struct_column.add_sub_column(int_child); struct_column.add_sub_column(string_child); - EXPECT_TRUE(Segment::get_default_value_field(struct_column, &field) - .is()); + Field struct_field; + ASSERT_TRUE(Segment::get_default_value_field(struct_column, &struct_field).ok()); + const auto& struct_value = struct_field.get(); + ASSERT_EQ(2, struct_value.size()); + EXPECT_EQ(99, struct_value[0].get()); + EXPECT_EQ("doris", std::string_view(struct_value[1].get())); } TEST_F(ConstantColumnIteratorTest, VariantConstantMaterializesSubcolumnsAndZoneMap) { From 0d9f0c992a87a85708cbfd7a25baf8de6aba0b8c Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Tue, 13 Oct 2026 19:56:56 +0800 Subject: [PATCH 6/6] [fix](complex) Fix pruned reads, partial-update NULL and replay of complex defaults ### What problem does this PR solve? Issue Number: None Related PR: #63528 Problem Summary: Follow-up fixes for literal defaults of ARRAY/MAP/STRUCT columns. 1. Nested column pruning reads only the accessed STRUCT fields, so after `ADD COLUMN s STRUCT DEFAULT '{7, "x"}'` on a table with old rows, `SELECT struct_element(s, 'b')` created a STRUCT destination while the old segment's constant default was parsed with the declared type and failed with `Cannot insert value of different size into struct`; STRUCT items of an ARRAY and STRUCT values of a MAP failed the same way. `Segment::new_column_iterator` now receives the materialized read type (`ReadSchema::data_type`) and, when access paths exist for the column, projects a `ConstantColumnIterator`'s Field onto it (`ConstantColumnIterator::project_to_read_type`): STRUCT fields are selected by the read type's names, ARRAY items and MAP values recurse, keys, scalars, NULL and the `{}` default are kept. 2. A partial update of a new key fills omitted columns through the BE JSON parser (`default_from_string`), whose nullable SerDe only recognizes a lower case `null`, so the canonical `[NULL]` text was stored as `["NULL"]`. The canonical text now renders a nested NULL as `null`, which every BE consumer of the text accepts. 3. A nested VARCHAR(n)/CHAR(n) value longer than n was truncated by the cast to VARCHAR(n); it is now rejected with `Default value is too long`, like the scalar default check. 4. `ARRAY DEFAULT '["-0"]'` stored `[-0.0]`, but replaying the SHOW CREATE TABLE text parsed `-0.0` through BigDecimal and stored `[0.0]`. A SQL numeric literal has no negative zero, so a float zero is always rendered as `0.0`. ### Release note None ### Check List (For Author) - Test: Unit Test / Regression test - BE UT: `ConstantColumnIteratorTest.*` (new `ComplexDefaultIsProjectedToPrunedReadType`), `*SegmentIteratorExprZonemap*`, `*IndexBuilder*` - FE UT: `ColumnDefinitionTest` - Regression: `datatype_p0/complex_types/test_complex_default_value` (pruned reads of light-added defaults, MoW partial update of a new key, VARCHAR/CHAR length rejects, negative zero replay) - Behavior changed: Yes (nested NULL in a complex default is stored as `null`; overlength nested VARCHAR/CHAR defaults are rejected; a float `-0.0` default is stored as `0.0`) - Does this need documentation: No --- .../storage/iterator/vgeneric_iterators.cpp | 5 +- be/src/storage/segment/column_reader.cpp | 89 ++++++++++++ be/src/storage/segment/column_reader.h | 7 + be/src/storage/segment/segment.cpp | 18 ++- be/src/storage/segment/segment.h | 5 +- be/src/storage/segment/segment_iterator.cpp | 10 +- be/src/storage/tablet/base_tablet.cpp | 3 +- be/test/storage/index/index_builder_test.cpp | 4 +- .../segment/constant_column_iterator_test.cpp | 136 ++++++++++++++++++ .../segment_iterator_expr_zonemap_test.cpp | 13 +- .../info/ComplexTypeDefaultValue.java | 63 +++++--- .../commands/info/ColumnDefinitionTest.java | 29 +++- .../test_complex_default_value.out | 18 ++- .../test_complex_default_value.groovy | 64 ++++++++- 14 files changed, 417 insertions(+), 47 deletions(-) diff --git a/be/src/storage/iterator/vgeneric_iterators.cpp b/be/src/storage/iterator/vgeneric_iterators.cpp index fa75c4bb6382e3..c3fba014d5fe7e 100644 --- a/be/src/storage/iterator/vgeneric_iterators.cpp +++ b/be/src/storage/iterator/vgeneric_iterators.cpp @@ -48,8 +48,9 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) { const auto* column_desc = _schema.column(ordinal); auto unique_id = column_desc->unique_id(); if (_column_iterators_map.count(unique_id) < 1) { - RETURN_IF_ERROR(_segment->new_column_iterator( - *column_desc, &_column_iterators_map[unique_id], &opts)); + RETURN_IF_ERROR( + _segment->new_column_iterator(*column_desc, _schema.data_type(ordinal), + &_column_iterators_map[unique_id], &opts)); } _column_iterators.push_back(_column_iterators_map[unique_id].get()); } diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 842161f64d1838..f0cc7cec319d47 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -43,8 +43,11 @@ #include "core/column/column_vector.h" #include "core/custom_allocator.h" #include "core/data_type/data_type_agg_state.h" +#include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "core/decimal12.h" #include "core/string_ref.h" @@ -740,6 +743,92 @@ Status ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicat return Status::OK(); } +namespace { + +// Keep only the nested values of `field` that `read_type` materializes. FE nested pruning keeps +// the declared field order, never prunes MAP keys and leaves scalars untouched, so a STRUCT is +// rebuilt from the read field names while ARRAY items and MAP values recurse. +Status project_field_to_read_type(const DataTypePtr& declared_type, const DataTypePtr& read_type, + Field& field) { + switch (field.get_type()) { + case PrimitiveType::TYPE_ARRAY: { + const auto& declared_item = + assert_cast(remove_nullable(declared_type).get()) + ->get_nested_type(); + const auto& read_item = assert_cast(remove_nullable(read_type).get()) + ->get_nested_type(); + Array items; + items.reserve(field.get().size()); + for (const auto& item : field.get()) { + Field projected = item; + RETURN_IF_ERROR(project_field_to_read_type(declared_item, read_item, projected)); + items.push_back(std::move(projected)); + } + field = Field::create_field(std::move(items)); + return Status::OK(); + } + case PrimitiveType::TYPE_MAP: { + const auto& declared_value = + assert_cast(remove_nullable(declared_type).get()) + ->get_value_type(); + const auto& read_value = + assert_cast(remove_nullable(read_type).get())->get_value_type(); + // A MAP Field holds its keys and its values as two ARRAY Fields. + const auto& map = field.get(); + DORIS_CHECK_EQ(map.size(), 2); + Array values; + values.reserve(map[1].get().size()); + for (const auto& value : map[1].get()) { + Field projected = value; + RETURN_IF_ERROR(project_field_to_read_type(declared_value, read_value, projected)); + values.push_back(std::move(projected)); + } + field = Field::create_field( + Map {map[0], Field::create_field(std::move(values))}); + return Status::OK(); + } + case PrimitiveType::TYPE_STRUCT: { + const auto& tuple = field.get(); + // The `{}` default means every field takes its own default, which ColumnStruct::insert + // applies to whatever fields the destination keeps. + if (tuple.empty()) { + return Status::OK(); + } + const auto* declared_struct = + assert_cast(remove_nullable(declared_type).get()); + const auto* read_struct = + assert_cast(remove_nullable(read_type).get()); + DORIS_CHECK_EQ(tuple.size(), declared_struct->get_elements().size()); + Struct projected_tuple; + const size_t read_fields = read_struct->get_elements().size(); + projected_tuple.reserve(read_fields); + for (size_t i = 0; i < read_fields; ++i) { + const auto position = + declared_struct->try_get_position_by_name(read_struct->get_name_by_position(i)); + DORIS_CHECK(position.has_value()) + << "read struct field " << read_struct->get_name_by_position(i) + << " is not declared by " << declared_struct->get_name(); + Field projected = tuple[*position]; + RETURN_IF_ERROR(project_field_to_read_type(declared_struct->get_element(*position), + read_struct->get_element(i), projected)); + projected_tuple.push_back(std::move(projected)); + } + field = Field::create_field(std::move(projected_tuple)); + return Status::OK(); + } + default: + // A scalar or NULL has no nested value to prune. + return Status::OK(); + } +} + +} // namespace + +Status ConstantColumnIterator::project_to_read_type(const DataTypePtr& declared_type, + const DataTypePtr& read_type) { + return project_field_to_read_type(declared_type, read_type, _value); +} + Status ConstantColumnReader::match_condition(const AndBlockColumnPredicate* col_predicates, bool* matched) const { ZoneMap zone_map; diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index dfbd236c3d3248..74f2a867286ad7 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -962,6 +962,13 @@ class ConstantColumnIterator : public ColumnIterator { ConstantColumnIterator() = delete; explicit ConstantColumnIterator(Field value) : _value(std::move(value)) {} + // Nested column pruning materializes `read_type`, which keeps only the requested STRUCT fields + // of `declared_type` at any nesting depth. For example, after `ADD COLUMN s STRUCT DEFAULT '{7, "x"}'`, `struct_element(s, 'b')` reads old segments into a + // STRUCT column, so the default parsed with the declared type must become {"x"} + // before it is inserted; a file iterator drops its pruned sub-iterators instead. + Status project_to_read_type(const DataTypePtr& declared_type, const DataTypePtr& read_type); + Status seek_to_ordinal(ordinal_t ord_idx) override { _current_rowid = ord_idx; return Status::OK(); diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 536b7baa366db4..0b103cd822313d 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -928,7 +928,7 @@ Status Segment::new_constant_iterator(const TabletColumn& tablet_column, // in the new schema column c's cid == 2 // but in the old schema column b's cid == 2 // but they are not the same column -Status Segment::new_column_iterator(const TabletColumn& tablet_column, +Status Segment::new_column_iterator(const TabletColumn& tablet_column, const DataTypePtr& read_type, std::unique_ptr* iter, const StorageReadOptions* opt, const std::unordered_map* @@ -979,6 +979,15 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column, ? opt->predicate_access_paths.at(unique_id) : TColumnAccessPaths {}; + // A column absent from this segment was parsed from its default with the declared + // type, while the destination column is built from the pruned `read_type`. For + // example, `struct_element(s, 'b')` reads a light-added `s STRUCT + // DEFAULT '{7, "x"}'` into STRUCT, so the constant must become {"x"}. + if (auto* constant_iter = dynamic_cast(iter->get()); + constant_iter != nullptr) { + RETURN_IF_ERROR(constant_iter->project_to_read_type(tablet_column.get_vec_type(), + read_type)); + } // set column name to apply access paths. (*iter)->set_column_name(tablet_column.name()); RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths)); @@ -1428,7 +1437,8 @@ Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDesc MutableColumnPtr file_storage_column = storage_type->create_column(); if (iterator_hint == nullptr) { - RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR(new_column_iterator(column, slot->type(), &iterator_hint, + &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR( @@ -1451,8 +1461,8 @@ Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDesc DORIS_CHECK(variant_v2_type != nullptr); } if (iterator_hint == nullptr) { - RETURN_IF_ERROR( - new_column_iterator(read_column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR(new_column_iterator(read_column, slot->type(), &iterator_hint, + &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR(iterator_hint->read_by_rowids(row_ids.data(), row_ids.size(), result)); diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index dfdb88ab72cdd7..d3ec5c7ab7798a 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -155,8 +155,11 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd MOCK_FUNCTION uint32_t num_rows() const { return _num_rows; } + // `read_type` is the type the caller materializes the column into. It equals the declared + // type unless nested column pruning requested a subset of STRUCT fields, in which case a + // constant default parsed with the declared type is projected onto it. // if variant_sparse_column_cache is nullptr, means the sparse column cache is not used - Status new_column_iterator(const TabletColumn& tablet_column, + Status new_column_iterator(const TabletColumn& tablet_column, const DataTypePtr& read_type, std::unique_ptr* iter, const StorageReadOptions* opt, const std::unordered_map* variant_sparse_column_cache = nullptr); diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 76c57993424e03..f07c2afa7cd925 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -797,8 +797,8 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra continue; } std::unique_ptr iter; - RETURN_IF_ERROR( - _segment->new_column_iterator(*col, &iter, &_opts, &_variant_sparse_column_cache)); + RETURN_IF_ERROR(_segment->new_column_iterator(*col, _seek_schema->data_type(i), &iter, + &_opts, &_variant_sparse_column_cache)); ColumnIteratorOptions iter_opts { .use_page_cache = _opts.use_page_cache, .file_reader = _file_reader.get(), @@ -1904,9 +1904,9 @@ Status SegmentIterator::_init_column_iterators() { } if (_column_iterators[cid] == nullptr) { - RETURN_IF_ERROR(_segment->new_column_iterator(*_schema->column(cid), - &_column_iterators[cid], &_opts, - &_variant_sparse_column_cache)); + RETURN_IF_ERROR(_segment->new_column_iterator( + *_schema->column(cid), _schema->data_type(cid), &_column_iterators[cid], &_opts, + &_variant_sparse_column_cache)); ColumnIteratorOptions iter_opts { .use_page_cache = _opts.use_page_cache, // If the col is predicate column, then should read the last page to check diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index b3bbf336ab6de6..eeb77c9cf5360d 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -119,7 +119,8 @@ Status _init_segment_column_iterator(const BetaRowsetSharedPtr& rowset, if (input_io_ctx != nullptr) { opts.io_ctx = *input_io_ctx; } - RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, &opts)); + RETURN_IF_ERROR(segment->new_column_iterator(target_column, target_column.get_vec_type(), + column_iterator, &opts)); auto io_ctx = opts.io_ctx; io_ctx.reader_type = ReaderType::READER_QUERY; io_ctx.file_cache_stats = &stats.file_cache_stats; diff --git a/be/test/storage/index/index_builder_test.cpp b/be/test/storage/index/index_builder_test.cpp index 3838529874bac6..9159522395199e 100644 --- a/be/test/storage/index/index_builder_test.cpp +++ b/be/test/storage/index/index_builder_test.cpp @@ -729,7 +729,9 @@ class IndexBuilderCommitTsoTest : public IndexBuilderTest, options.commit_tso = rowset->rowset_meta()->commit_tso(); options.io_ctx.reader_type = ReaderType::READER_QUERY; segment_v2::ColumnIteratorUPtr iter; - st = segment->new_column_iterator(rowset->tablet_schema()->column(2), &iter, &options); + st = segment->new_column_iterator(rowset->tablet_schema()->column(2), + rowset->tablet_schema()->column(2).get_vec_type(), &iter, + &options); // Published singleton rowsets with absent or [-1, -1] TSOs must reject reads, // even though index build/drop must preserve that metadata unchanged. if (!tso.has_value() || tso->end_tso() == -1) { diff --git a/be/test/storage/segment/constant_column_iterator_test.cpp b/be/test/storage/segment/constant_column_iterator_test.cpp index 094c6eea767a2b..77ea461f897dd0 100644 --- a/be/test/storage/segment/constant_column_iterator_test.cpp +++ b/be/test/storage/segment/constant_column_iterator_test.cpp @@ -35,6 +35,11 @@ #include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" #include "core/field.h" #include "core/value/bitmap_value.h" #include "core/value/hll.h" @@ -1028,6 +1033,137 @@ TEST_F(ConstantColumnIteratorTest, NonEmptyComplexTextDefaultsAreParsed) { EXPECT_EQ("doris", std::string_view(struct_value[1].get())); } +TEST_F(ConstantColumnIteratorTest, ComplexDefaultIsProjectedToPrunedReadType) { + // Case: nested column pruning reads `struct_element(s, 'text')` of a light-added + // STRUCT DEFAULT '{99, "doris"}' into a STRUCT column. + // The constant parsed with the declared type must keep only the read fields, also for STRUCT + // items of an ARRAY and STRUCT values of a MAP, while the `{}` default stays empty so the + // destination fills its own defaults. + TabletColumn int_child; + int_child.set_name("number"); + int_child.set_type(FieldType::OLAP_FIELD_TYPE_INT); + int_child.set_is_nullable(false); + + TabletColumn string_child; + string_child.set_name("text"); + string_child.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + string_child.set_is_nullable(false); + + TabletColumn struct_column; + struct_column.set_name("s"); + struct_column.set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); + struct_column.set_is_nullable(false); + struct_column.set_default_value("{99,\"doris\"}"); + struct_column.add_sub_column(int_child); + struct_column.add_sub_column(string_child); + // FE prunes the slot type to the accessed fields, in declared order, keeping nullability. + DataTypePtr read_struct = std::make_shared( + DataTypes {std::make_shared()}, Strings {"text"}); + + Field struct_default; + auto st = Segment::get_default_value_field(struct_column, &struct_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator struct_iterator(struct_default); + st = struct_iterator.project_to_read_type(struct_column.get_vec_type(), read_struct); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr struct_dst = read_struct->create_column(); + size_t rows = 2; + bool has_null = true; + st = struct_iterator.next_batch(&rows, struct_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_struct = assert_cast(struct_dst.get()); + ASSERT_EQ(1, materialized_struct->tuple_size()); + ASSERT_EQ(2, materialized_struct->size()); + EXPECT_EQ("doris", materialized_struct->get_column(0).get_data_at(0).to_string()); + EXPECT_EQ("doris", materialized_struct->get_column(0).get_data_at(1).to_string()); + + // Projection to the declared type itself keeps every field. + ConstantColumnIterator full_iterator(struct_default); + st = full_iterator.project_to_read_type(struct_column.get_vec_type(), + struct_column.get_vec_type()); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr full_dst = struct_column.get_vec_type()->create_column(); + rows = 1; + st = full_iterator.next_batch(&rows, full_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* full_struct = assert_cast(full_dst.get()); + ASSERT_EQ(2, full_struct->tuple_size()); + EXPECT_EQ(99, assert_cast(full_struct->get_column(0)).get_element(0)); + EXPECT_EQ("doris", full_struct->get_column(1).get_data_at(0).to_string()); + + // ARRAY: every item is projected. + TabletColumn array_column; + array_column.set_name("arr"); + array_column.set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); + array_column.set_is_nullable(false); + array_column.set_default_value("[{1,\"p\"},{2,\"q\"}]"); + array_column.add_sub_column(struct_column); + DataTypePtr read_array = std::make_shared(make_nullable(read_struct)); + Field array_default; + st = Segment::get_default_value_field(array_column, &array_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator array_iterator(array_default); + st = array_iterator.project_to_read_type(array_column.get_vec_type(), read_array); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr array_dst = read_array->create_column(); + rows = 1; + st = array_iterator.next_batch(&rows, array_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_array = assert_cast(array_dst.get()); + ASSERT_EQ(2, materialized_array->size_at(0)); + const auto& array_items = assert_cast( + assert_cast(materialized_array->get_data()).get_nested_column()); + ASSERT_EQ(1, array_items.tuple_size()); + EXPECT_EQ("p", array_items.get_column(0).get_data_at(0).to_string()); + EXPECT_EQ("q", array_items.get_column(0).get_data_at(1).to_string()); + + // MAP: keys are never pruned, every value is projected. + TabletColumn map_column; + map_column.set_name("m"); + map_column.set_type(FieldType::OLAP_FIELD_TYPE_MAP); + map_column.set_is_nullable(false); + map_column.set_default_value("{\"k\":{3,\"r\"}}"); + map_column.add_sub_column(string_child); + map_column.add_sub_column(struct_column); + DataTypePtr read_map = std::make_shared( + make_nullable(std::make_shared()), make_nullable(read_struct)); + Field map_default; + st = Segment::get_default_value_field(map_column, &map_default); + ASSERT_TRUE(st.ok()) << st; + ConstantColumnIterator map_iterator(map_default); + st = map_iterator.project_to_read_type(map_column.get_vec_type(), read_map); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr map_dst = read_map->create_column(); + rows = 1; + st = map_iterator.next_batch(&rows, map_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* materialized_map = assert_cast(map_dst.get()); + ASSERT_EQ(1, materialized_map->size_at(0)); + EXPECT_EQ("k", materialized_map->get_keys().get_data_at(0).to_string()); + const auto& map_values = assert_cast( + assert_cast(materialized_map->get_values()).get_nested_column()); + ASSERT_EQ(1, map_values.tuple_size()); + EXPECT_EQ("r", map_values.get_column(0).get_data_at(0).to_string()); + + // `{}` means every field takes its own default, so the pruned destination fills itself. + struct_column.set_default_value("{}"); + Field empty_default; + st = Segment::get_default_value_field(struct_column, &empty_default); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(empty_default.get().empty()); + ConstantColumnIterator empty_iterator(empty_default); + st = empty_iterator.project_to_read_type(struct_column.get_vec_type(), read_struct); + ASSERT_TRUE(st.ok()) << st; + MutableColumnPtr empty_dst = read_struct->create_column(); + rows = 2; + st = empty_iterator.next_batch(&rows, empty_dst, &has_null); + ASSERT_TRUE(st.ok()) << st; + const auto* empty_struct = assert_cast(empty_dst.get()); + ASSERT_EQ(1, empty_struct->tuple_size()); + ASSERT_EQ(2, empty_struct->size()); + EXPECT_EQ("", empty_struct->get_column(0).get_data_at(1).to_string()); +} + TEST_F(ConstantColumnIteratorTest, VariantConstantMaterializesSubcolumnsAndZoneMap) { // Case: a VARIANT object constant contains both a direct child and a nested object. Repeating // it must preserve both values, while text VARIANT defaults remain unsupported. diff --git a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp index ed490410d1a632..e90f80ee8ff120 100644 --- a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp +++ b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp @@ -322,7 +322,9 @@ class SegmentIteratorExprZonemapTest : public testing::Test { void read_column(const std::shared_ptr& segment, int32_t cid, const StorageReadOptions& read_options, MutableColumnPtr* dst) { ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(_tablet_schema->column(cid), &iter, &read_options); + auto st = segment->new_column_iterator(_tablet_schema->column(cid), + _tablet_schema->column(cid).get_vec_type(), &iter, + &read_options); ASSERT_TRUE(st.ok()) << st; ASSERT_NE(nullptr, iter); @@ -351,7 +353,9 @@ class SegmentIteratorExprZonemapTest : public testing::Test { void expect_missing_commit_tso_error(const std::shared_ptr& segment, int32_t cid, const StorageReadOptions& read_options) { ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(_tablet_schema->column(cid), &iter, &read_options); + auto st = segment->new_column_iterator(_tablet_schema->column(cid), + _tablet_schema->column(cid).get_vec_type(), &iter, + &read_options); EXPECT_TRUE(st.is()) << st; EXPECT_NE(st.to_string().find("requires a valid commit tso"), std::string::npos) << st; } @@ -887,7 +891,7 @@ TEST_F(SegmentIteratorExprZonemapTest, MissingVariantDocBucketsDefaultToNull) { auto column = variant_util::create_doc_value_compaction_column(_tablet_schema->column(1), bucket); ColumnIteratorUPtr iter; - auto st = segment->new_column_iterator(column, &iter, &read_options); + auto st = segment->new_column_iterator(column, column.get_vec_type(), &iter, &read_options); ASSERT_TRUE(st.ok()) << st; ColumnIteratorOptions opts; opts.stats = &_stats; @@ -948,7 +952,8 @@ TEST_F(SegmentIteratorExprZonemapTest, MissingVariantSeparatesValueAndPhysicalRe // Reading a generated child path must use the missing root's logical NULL value. It must not // attempt to cast the ConstantColumnReader to VariantColumnReader or search sparse metadata. ColumnIteratorUPtr path_iterator; - st = segment->new_column_iterator(added_path, &path_iterator, &read_options); + st = segment->new_column_iterator(added_path, declared_path_type, &path_iterator, + &read_options); ASSERT_TRUE(st.ok()) << st; ASSERT_NE(nullptr, path_iterator); ColumnIteratorOptions iterator_options; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java index f0e0c8e98d1fe1..9398e8a4fc7945 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java @@ -27,6 +27,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -34,8 +36,10 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.coercion.CharacterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -56,12 +60,13 @@ * (see {@link LiteralBuilder}), and every nested value is then cast to the declared nested type, so a * type mismatch is rejected at DDL time instead of when old rows are read. The * canonical text only contains plain nested values (unquoted numbers, double quoted strings, nested - * brackets and NULL). BE parses the stored text in two places that must agree: the complex SerDe - * {@code from_fe_string} used by the default value iterator and schema change, and the - * string-to-complex cast used when INSERT fills an unmentioned column. Neither of them decodes - * escape sequences, and the DDL parser keeps the text of a default value verbatim when SHOW CREATE - * TABLE output is replayed, so string values containing quotes or backslashes are rejected instead - * of stored. + * brackets and lower case null). BE parses the stored text in three places that must agree: the + * complex SerDe {@code from_fe_string} used by the default value iterator and schema change, the + * string-to-complex cast used when INSERT fills an unmentioned column, and the JSON parser + * {@code default_from_string} that fills the omitted columns of a partial update, which only + * recognizes a lower case nested null. None of them decodes escape sequences, and the DDL parser + * keeps the text of a default value verbatim when SHOW CREATE TABLE output is replayed, so string + * values containing quotes or backslashes are rejected instead of stored. */ public class ComplexTypeDefaultValue { private ComplexTypeDefaultValue() { @@ -114,7 +119,7 @@ private static String capitalize(String value) { private static String render(Literal literal, DataType type) throws AnalysisException { if (literal instanceof NullLiteral) { - return "NULL"; + return "null"; } if (type.isArrayType()) { if (!(literal instanceof ArrayLiteral)) { @@ -167,31 +172,53 @@ private static String render(Literal literal, DataType type) throws AnalysisExce } return joiner.toString(); } - return renderScalar((Literal) literal.checkedCastTo(type), type); + return renderScalar(literal, type); } private static String renderScalar(Literal literal, DataType type) throws AnalysisException { + if (type.isStringLikeType()) { + // The cast to VARCHAR(n) silently truncates a longer value, while a scalar VARCHAR(n) column + // rejects such a default, so cast to STRING and check the length against the declared type. + String value = ((Literal) literal.checkedCastTo(StringType.INSTANCE)).getStringValue(); + int maxLength = ((CharacterType) type).getLen(); + if (maxLength >= 0 && value.length() > maxLength) { + throw new AnalysisException("Default value is too long: " + value); + } + return renderQuoted(literal, value); + } + literal = (Literal) literal.checkedCastTo(type); if (type.isBooleanType()) { return ((BooleanLiteral) literal).getValue() ? "1" : "0"; } if (type.isNumericType()) { - if (type.isFloatLikeType() && !Double.isFinite(((Number) literal.getValue()).doubleValue())) { - throw new AnalysisException(literal.toSql() + " is not a finite number"); + if (type.isFloatLikeType()) { + double value = ((Number) literal.getValue()).doubleValue(); + if (!Double.isFinite(value)) { + throw new AnalysisException(literal.toSql() + " is not a finite number"); + } + // A SQL numeric literal has no negative zero: replaying the "-0.0" of SHOW CREATE TABLE + // output parses it through BigDecimal and stores 0.0, so render -0.0 as 0.0 from the start. + if (value == 0) { + literal = type.isFloatType() ? new FloatLiteral(0.0f) : new DoubleLiteral(0.0); + } } return literal.getStringValue(); } - if (type.isStringLikeType() || type.isDateType() || type.isDateV2Type() || type.isDateTimeType() - || type.isDateTimeV2Type() || type.isIPType()) { - String value = literal.getStringValue(); - if (value.indexOf('"') >= 0 || value.indexOf('\'') >= 0 || value.indexOf('\\') >= 0) { - throw new AnalysisException("string value " + literal.toSql() - + " must not contain quote or backslash"); - } - return "\"" + value + "\""; + if (type.isDateType() || type.isDateV2Type() || type.isDateTimeType() || type.isDateTimeV2Type() + || type.isIPType()) { + return renderQuoted(literal, literal.getStringValue()); } throw new AnalysisException(type.toSql() + " is not supported as a nested type of a default value"); } + private static String renderQuoted(Literal literal, String value) throws AnalysisException { + if (value.indexOf('"') >= 0 || value.indexOf('\'') >= 0 || value.indexOf('\\') >= 0) { + throw new AnalysisException("string value " + literal.toSql() + + " must not contain quote or backslash"); + } + return "\"" + value + "\""; + } + /** * Builds array and map literals from their items as written. * diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index 5c2e4100eff948..dd1b0f6d0d5194 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -24,10 +24,12 @@ import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.HllType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.JsonType; @@ -37,6 +39,7 @@ import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.UuidType; +import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.VariantType; import org.apache.doris.qe.SessionVariable; @@ -171,7 +174,9 @@ public void testComplexTypeLiteralDefaultValue() { ArrayType intArray = ArrayType.of(IntegerType.INSTANCE); assertCanonicalDefaultValue(intArray, "[]", "[]"); assertCanonicalDefaultValue(intArray, "[1, 2]", "[1, 2]"); - assertCanonicalDefaultValue(intArray, "[NULL, nUlL, 5]", "[NULL, NULL, 5]"); + // nested NULL is rendered in lower case: the BE JSON parser that fills the omitted columns of a + // partial update only recognizes `null`, and `NULL` would be stored as the string "NULL" + assertCanonicalDefaultValue(intArray, "[NULL, nUlL, 5]", "[null, null, 5]"); assertCanonicalDefaultValue(intArray, "[\"7\", 1e3]", "[7, 1000]"); assertRejectsDefaultValue(intArray, "{}", "only supports array literals or DEFAULT NULL"); assertRejectsDefaultValue(intArray, "[1 + 1]", "only supports array literals or DEFAULT NULL"); @@ -191,6 +196,11 @@ public void testComplexTypeLiteralDefaultValue() { "[[DATEV2 \"2024-01-01\"], [1]]", "Invalid default value"); assertCanonicalDefaultValue(ArrayType.of(BooleanType.INSTANCE), "[true, false]", "[1, 0]"); assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[1.5, 2]", "[1.5, 2.0]"); + // a SQL numeric literal has no negative zero, so the canonical text never contains -0.0 + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[\"-0\", -0.0, \"-0.0\"]", + "[0.0, 0.0, 0.0]"); + assertCanonicalDefaultValue(ArrayType.of(FloatType.INSTANCE), "[\"-0\", -0.0]", "[0.0, 0.0]"); + assertCanonicalDefaultValue(ArrayType.of(DoubleType.INSTANCE), "[\"-1.5\"]", "[-1.5]"); assertCanonicalDefaultValue(ArrayType.of(DecimalV3Type.createDecimalV3Type(10, 2)), "[1.234]", "[1.23]"); assertCanonicalDefaultValue(ArrayType.of(StringType.INSTANCE), "['x,y', \"[z]\", \"{k:v}\", '']", "[\"x,y\", \"[z]\", \"{k:v}\", \"\"]"); @@ -200,11 +210,22 @@ public void testComplexTypeLiteralDefaultValue() { "must not contain quote or backslash"); assertRejectsDefaultValue(ArrayType.of(StringType.INSTANCE), "['a\\\\b']", "must not contain quote or backslash"); - assertCanonicalDefaultValue(ArrayType.of(intArray), "[[1], [], NULL]", "[[1], [], NULL]"); + assertCanonicalDefaultValue(ArrayType.of(intArray), "[[1], [], NULL]", "[[1], [], null]"); + // a nested VARCHAR(n)/CHAR(n) value longer than n is rejected like a scalar default, instead of + // being truncated by the cast to VARCHAR(n) + assertCanonicalDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[\"ab\", 1]", + "[\"ab\", \"1\"]"); + assertRejectsDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[\"abc\"]", + "Default value is too long: abc"); + assertRejectsDefaultValue(ArrayType.of(VarcharType.createVarcharType(2)), "[123]", + "Default value is too long: 123"); + assertCanonicalDefaultValue(ArrayType.of(CharType.createCharType(2)), "[\"ab\"]", "[\"ab\"]"); + assertRejectsDefaultValue(ArrayType.of(CharType.createCharType(2)), "[\"abc\"]", + "Default value is too long: abc"); MapType stringIntMap = MapType.of(StringType.INSTANCE, IntegerType.INSTANCE); assertCanonicalDefaultValue(stringIntMap, "{}", "{}"); - assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"b\": NULL}", "{\"a\":1, \"b\":NULL}"); + assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"b\": NULL}", "{\"a\":1, \"b\":null}"); assertCanonicalDefaultValue(stringIntMap, "{\"a\": 1, \"a\": 2}", "{\"a\":2}"); assertRejectsDefaultValue(stringIntMap, "[]", "only supports map literals or DEFAULT NULL"); assertRejectsDefaultValue(stringIntMap, "{\"a\": \"bad\"}", "Invalid default value"); @@ -225,7 +246,7 @@ public void testComplexTypeLiteralDefaultValue() { new StructField("f2", StringType.INSTANCE, true, ""))); assertCanonicalDefaultValue(structType, "{}", "{}"); assertCanonicalDefaultValue(structType, "{1, \"a\"}", "{1, \"a\"}"); - assertCanonicalDefaultValue(structType, "{NULL, 2}", "{NULL, \"2\"}"); + assertCanonicalDefaultValue(structType, "{NULL, 2}", "{null, \"2\"}"); assertRejectsDefaultValue(structType, "{\"f1\": 1, \"f2\": \"a\"}", "only supports struct literals or DEFAULT NULL"); assertRejectsDefaultValue(structType, "{1}", "struct literal has 1 fields but the column has 2"); diff --git a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out index bd638b931e7e2d..71c9190bc7f824 100644 --- a/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out +++ b/regression-test/data/datatype_p0/complex_types/test_complex_default_value.out @@ -1,19 +1,23 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !literal_default -- -1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] ["ab", "1"] [0, 0, -1.5] -- !like_default -- -1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] +1 [] [1, 2] {} {"a":10, "b":20} {"f1":null, "f2":null} {"f1":7, "f2":"x"} [null, null, 5] {"upper":null, "mixed":null, "value":6} {"f1":null, "f2":null, "f3":7} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] ["ab", "1"] [0, 0, -1.5] -- !null_default -- 1 \N \N \N \N \N -- !alter_default -- -1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} -- !alter_default_after_insert -- -1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] -2 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] +1 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} +2 [3, 4] {"z":9} {"f1":8, "f2":"y"} ["2024-01-01", "2024-02-02"] [1000, 7] [1, 0] [1.23] ["x,y", "[z]", "{k:v}", "null", ""] {"a":2} [[1], [], null] {"f1":null, "f2":null} [{"f1":1, "f2":"p"}, {"f1":2, "f2":"q"}] {"k":{"f1":3, "f2":"r"}} + +-- !alter_default_pruned -- +1 y \N ["p", "q"] r 2 false +2 y \N ["p", "q"] r 2 false -- !alter_direct_default -- 1 [3, 4] {"a":2} {"f1":8, "f2":["2024-01-01"]} ["x,y", ""] @@ -32,3 +36,7 @@ 2 [1000, 7] 2 [5] +-- !partial_update_default -- +1 [null, "null", "x"] true {"a":null, "b":"y"} {"f1":null, "f2":null} true {"f1":7, "f2":"x"} [1, 2] +2 [null, "null", "x"] true {"a":null, "b":"y"} {"f1":null, "f2":null} true {"f1":7, "f2":"x"} [1, 2] + diff --git a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy index 8dffa1b6919f03..477ed86dbb2707 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_complex_default_value.groovy @@ -24,6 +24,7 @@ suite("test_complex_default_value") { sql "DROP TABLE IF EXISTS test_complex_default_value_bad_alter" sql "DROP TABLE IF EXISTS test_complex_default_value_replace_value" sql "DROP TABLE IF EXISTS test_complex_default_value_mapping" + sql "DROP TABLE IF EXISTS test_complex_default_value_partial_update" def createTableRejects = { String columnDef, String message -> test { @@ -64,6 +65,11 @@ suite("test_complex_default_value") { createTableRejects.call("""STRUCT> DEFAULT '{[DATEV2 "2024-01-01", 1]}'""", "Invalid default value") // map keys that only collide after the cast to the key type are rejected createTableRejects.call("""MAP DEFAULT '{"01": 1, "1": 2}'""", "map key 1 is repeated after casting to INT") + // a nested value longer than the declared VARCHAR(n)/CHAR(n) is rejected like a scalar default instead of + // being truncated + createTableRejects.call("""ARRAY DEFAULT '["abc"]'""", "Default value is too long: abc") + createTableRejects.call("""ARRAY DEFAULT '["abc"]'""", "Default value is too long: abc") + createTableRejects.call("""STRUCT DEFAULT '{123}'""", "Default value is too long: 123") // nested string values are stored as plain double quoted text, so quotes and backslashes are rejected createTableRejects.call("""ARRAY DEFAULT '["a""b"]'""", "must not contain quote or backslash") @@ -104,6 +110,10 @@ suite("test_complex_default_value") { sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v MAP DEFAULT '{"a": DATEV2 "2024-01-01", "b": 1}'""" exception "Invalid default value" } + test { + sql """ALTER TABLE test_complex_default_value_bad_alter ADD COLUMN v ARRAY DEFAULT '["abc"]'""" + exception "Default value is too long: abc" + } sql """ CREATE TABLE test_complex_default_value_literal ( @@ -123,7 +133,9 @@ suite("test_complex_default_value") { arr_decimal ARRAY DEFAULT '[1.234]', arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', - arr_nested ARRAY> DEFAULT '[[1], [], NULL]' + arr_nested ARRAY> DEFAULT '[[1], [], NULL]', + arr_varchar ARRAY DEFAULT '["ab", 1]', + arr_negative_zero ARRAY DEFAULT '["-0", -0.0, "-1.5"]' ) DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 @@ -146,6 +158,11 @@ suite("test_complex_default_value") { assertTrue(createSql.contains('`arr_bool` array NULL DEFAULT "[1, 0]"')) assertTrue(createSql.contains('`map_repeated_key` map NULL DEFAULT \'{"a":2}\'')) assertTrue(createSql.contains('`arr_string` array NULL DEFAULT \'["x,y", "[z]", "{k:v}", "null", ""]\'')) + // nested NULL is stored in lower case, the only spelling every BE consumer of the text accepts + assertTrue(createSql.contains('`arr_nested_null` array NULL DEFAULT "[null, null, 5]"')) + assertTrue(createSql.contains('`struct_nested_null` struct NULL DEFAULT "{null, null, 7}"')) + // a SQL numeric literal has no negative zero, so -0.0 is stored as 0.0 and replays unchanged + assertTrue(createSql.contains('`arr_negative_zero` array NULL DEFAULT "[0.0, 0.0, -1.5]"')) sql "CREATE TABLE test_complex_default_value_like LIKE test_complex_default_value_literal" sql "INSERT INTO test_complex_default_value_like(k) VALUES (1)" order_qt_like_default """ @@ -198,7 +215,10 @@ suite("test_complex_default_value") { ADD COLUMN arr_decimal ARRAY DEFAULT '[1.234]', ADD COLUMN arr_string ARRAY DEFAULT '["x,y", "[z]", "{k:v}", "null", ""]', ADD COLUMN map_repeated_key MAP DEFAULT '{"a": 1, "a": 2}', - ADD COLUMN arr_nested ARRAY> DEFAULT '[[1], [], NULL]' + ADD COLUMN arr_nested ARRAY> DEFAULT '[[1], [], NULL]', + ADD COLUMN struct_empty STRUCT DEFAULT '{}', + ADD COLUMN arr_struct ARRAY> DEFAULT '[{1, "p"}, {2, "q"}]', + ADD COLUMN map_struct MAP> DEFAULT '{"k": {3, "r"}}' """ waitForSchemaChangeDone { sql """SHOW ALTER TABLE COLUMN WHERE IndexName='test_complex_default_value_alter_base' ORDER BY createtime DESC LIMIT 1""" @@ -213,6 +233,18 @@ suite("test_complex_default_value") { order_qt_alter_default_after_insert """ SELECT * FROM test_complex_default_value_alter_base ORDER BY k """ + // nested column pruning reads only the accessed STRUCT fields, so the default of an old row is + // projected onto the pruned STRUCT, also inside ARRAY items and MAP values + order_qt_alter_default_pruned """ + SELECT k, + struct_element(struct_added, 'f2'), + struct_element(struct_empty, 'f2'), + array_map(x -> struct_element(x, 'f2'), arr_struct), + struct_element(element_at(map_struct, 'k'), 'f2'), + size(arr_struct), + struct_added IS NULL + FROM test_complex_default_value_alter_base ORDER BY k + """ // direct schema change materializes the defaults into the rewritten rows sql """ @@ -303,4 +335,32 @@ suite("test_complex_default_value") { """ exception "map key 1 is repeated after casting to INT" } + + // a partial update of a new key fills the omitted complex columns from their defaults through the + // BE JSON default parser, which must read a nested null the same way as the default value iterator + sql """ + CREATE TABLE test_complex_default_value_partial_update ( + k INT, + v INT, + arr_nested_null ARRAY DEFAULT '[NULL, "null", "x"]', + map_nested_null MAP DEFAULT '{"a": NULL, "b": "y"}', + struct_nested_null STRUCT DEFAULT '{NULL, NULL}', + struct_literal STRUCT DEFAULT '{7, "x"}', + arr_literal ARRAY DEFAULT '[1, 2]' + ) + UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES('replication_num'='1', 'enable_unique_key_merge_on_write'='true') + """ + sql "INSERT INTO test_complex_default_value_partial_update(k, v) VALUES (1, 1)" + sql "SET enable_unique_key_partial_update = true" + sql "SET enable_insert_strict = false" + sql "INSERT INTO test_complex_default_value_partial_update(k) VALUES (2)" + sql "SET enable_unique_key_partial_update = false" + sql "SET enable_insert_strict = true" + order_qt_partial_update_default """ + SELECT k, arr_nested_null, array_contains(arr_nested_null, NULL), map_nested_null, + struct_nested_null, struct_element(struct_nested_null, 'f2') IS NULL, struct_literal, arr_literal + FROM test_complex_default_value_partial_update ORDER BY k + """ }