[fix](complex) Support literal defaults for complex columns

be/src/core/column/column_struct.cpp

@@ -120,6 +120,10 @@ void ColumnStruct::insert(const Field& x) {
     DCHECK_EQ(x.get_type(), PrimitiveType::TYPE_STRUCT);
     const auto& tuple = x.get<TYPE_STRUCT>();
     const size_t tuple_size = columns.size();
+    if (tuple.empty() && tuple_size != 0) {
+        insert_default();
+        return;
+    }
     if (tuple.size() != tuple_size) {
         throw doris::Exception(
                 ErrorCode::INTERNAL_ERROR,

be/src/core/data_type_serde/complex_type_deserialize_util.h

@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include "core/data_type_serde/data_type_nullable_serde.h"
 #include "core/data_type_serde/data_type_serde.h"
 #include "core/string_ref.h"
 
@@ -80,13 +81,37 @@ struct ComplexTypeDeserializeUtil {
 
     static bool is_null_string(const StringRef& str) {
         if (str.size == 4) {
-            // null
-            return str.data[0] == 'n' && str.data[1] == 'u' && str.data[2] == 'l' &&
-                   str.data[3] == 'l';
+            // SQL NULL literal is case-insensitive.
+            return (str.data[0] == 'n' || str.data[0] == 'N') &&
+                   (str.data[1] == 'u' || str.data[1] == 'U') &&
+                   (str.data[2] == 'l' || str.data[2] == 'L') &&
+                   (str.data[3] == 'l' || str.data[3] == 'L');
         }
         return false;
     }
 
+    static Status process_field(const DataTypeSerDeSPtr& serde, StringRef str, Field& field) {
+        str = str.trim_whitespace();
+        auto nullable_serde = std::dynamic_pointer_cast<DataTypeNullableSerDe>(serde);
+        if (is_null_string(str)) {
+            if (nullable_serde == nullptr) {
+                return Status::InvalidArgument(
+                        "NULL default is not allowed for non-nullable complex field");
+            }
+            field = Field::create_field<TYPE_NULL>(Null {});
+            return Status::OK();
+        }
+        auto str_without_quote = str.trim_quote();
+        auto nested_serde = serde;
+        if (nullable_serde != nullptr) {
+            // Complex default values need strict nested conversion. DataTypeNullableSerDe
+            // converts nested parse failures to NULL, which would silently accept
+            // type-mismatched default literals.
+            nested_serde = nullable_serde->get_nested_serde();
+        }
+        return nested_serde->from_fe_string(str_without_quote.to_string(), field);
+    }
+
     template <bool is_strict_mode>
     static Status process_column(const DataTypeSerDeSPtr& serde, IColumn& column, StringRef& str,
                                  const DataTypeSerDe::FormatOptions& options) {
@@ -107,4 +132,4 @@ struct ComplexTypeDeserializeUtil {
     }
 };
 
-} // namespace doris
+} // namespace doris

be/src/core/data_type_serde/data_type_array_serde.cpp

@@ -593,6 +593,42 @@ Status DataTypeArraySerDe::_from_string(StringRef& str, IColumn& column,
     return Status::OK();
 }
 
+Status DataTypeArraySerDe::from_fe_string(const std::string& str, Field& field) const {
+    StringRef slice(str);
+    slice = slice.trim_whitespace();
+    if (slice.empty()) {
+        return Status::InvalidArgument("slice is empty!");
+    }
+    if (slice.front() != '[') {
+        return Status::InvalidArgument("Array does not start with '[' character, found '{}'",
+                                       slice.to_string());
+    }
+    if (slice.back() != ']') {
+        return Status::InvalidArgument("Array does not end with ']' character, found '{}'",
+                                       slice.to_string());
+    }
+
+    Array array;
+    if (slice.size > 2) {
+        slice = slice.substring(1, slice.size - 2);
+        slice = slice.trim_whitespace();
+        if (!slice.empty()) {
+            FormatOptions options;
+            auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(
+                    slice, [&](char c) { return c == options.collection_delim; });
+            array.reserve(split_result.size());
+            for (auto& item : split_result) {
+                Field item_field;
+                RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field(
+                        nested_serde, item.element, item_field));
+                array.push_back(std::move(item_field));
+            }
+        }
+    }
+    field = Field::create_field<TYPE_ARRAY>(std::move(array));
+    return Status::OK();
+}
+
 Status DataTypeArraySerDe::from_string(StringRef& str, IColumn& column,
                                        const FormatOptions& options) const {
     return _from_string<false>(str, column, options);

be/src/core/data_type_serde/data_type_array_serde.h

@@ -54,6 +54,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
     Status from_string_strict_mode(StringRef& str, IColumn& column,
                                    const FormatOptions& options) const override;
 
+    Status from_fe_string(const std::string& str, Field& field) const override;
+
     Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
                                       FormatOptions& options) const override;
 

be/src/core/data_type_serde/data_type_map_serde.cpp

@@ -756,6 +756,68 @@ Status DataTypeMapSerDe::_from_string(StringRef& str, IColumn& column,
     return Status::OK();
 }
 
+Status DataTypeMapSerDe::from_fe_string(const std::string& str, Field& field) const {
+    StringRef slice(str);
+    slice = slice.trim_whitespace();
+    if (slice.empty()) {
+        return Status::InvalidArgument("slice is empty!");
+    }
+    if (slice.front() != '{') {
+        std::stringstream ss;
+        ss << slice.front() << '\'';
+        return Status::InvalidArgument("Map does not start with '{' character, found '" + ss.str());
+    }
+    if (slice.back() != '}') {
+        std::stringstream ss;
+        ss << slice.back() << '\'';
+        return Status::InvalidArgument("Map does not end with '}' character, found '" + ss.str());
+    }
+
+    Array keys;
+    Array values;
+    if (slice.size > 2) {
+        slice = slice.substring(1, slice.size - 2);
+        slice = slice.trim_whitespace();
+        if (!slice.empty()) {
+            FormatOptions options;
+            auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) {
+                return c == options.map_key_delim || c == options.collection_delim;
+            });
+            if (split_result.size() % 2 != 0) {
+                return Status::InvalidArgument("Map does not have even number of key-value pairs");
+            }
+            keys.reserve(split_result.size() / 2);
+            values.reserve(split_result.size() / 2);
+            for (int i = 0; i < split_result.size(); i += 2) {
+                if (split_result[i].delimiter != options.map_key_delim) {
+                    return Status::InvalidArgument(
+                            "Map key-value pair does not have map key delimiter");
+                }
+                if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) {
+                    return Status::InvalidArgument(
+                            "Map key-value pair does not have collection delimiter");
+                }
+
+                Field key_field;
+                Field value_field;
+                RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field(
+                        key_serde, split_result[i].element, key_field));
+                RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field(
+                        value_serde, split_result[i + 1].element, value_field));
+                keys.push_back(std::move(key_field));
+                values.push_back(std::move(value_field));
+            }
+        }
+    }
+
+    Map map;
+    map.reserve(2);
+    map.push_back(Field::create_field<TYPE_ARRAY>(std::move(keys)));
+    map.push_back(Field::create_field<TYPE_ARRAY>(std::move(values)));
+    field = Field::create_field<TYPE_MAP>(std::move(map));
+    return Status::OK();
+}
+
 Status DataTypeMapSerDe::from_string(StringRef& str, IColumn& column,
                                      const FormatOptions& options) const {
     return _from_string<false>(str, column, options);

be/src/core/data_type_serde/data_type_map_serde.h

@@ -47,6 +47,8 @@ class DataTypeMapSerDe : public DataTypeSerDe {
     Status from_string_strict_mode(StringRef& str, IColumn& column,
                                    const FormatOptions& options) const override;
 
+    Status from_fe_string(const std::string& str, Field& field) const override;
+
     Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
                                       FormatOptions& options) const override;
     Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx,

be/src/core/data_type_serde/data_type_struct_serde.cpp

@@ -734,6 +734,87 @@ Status DataTypeStructSerDe::_from_string(StringRef& str, IColumn& column,
     return Status::OK();
 }
 
+Status DataTypeStructSerDe::from_fe_string(const std::string& str, Field& field) const {
+    StringRef slice(str);
+    slice = slice.trim_whitespace();
+    if (slice.empty()) {
+        return Status::InvalidArgument("slice is empty!");
+    }
+    if (slice.front() != '{') {
+        std::stringstream ss;
+        ss << slice.front() << '\'';
+        return Status::InvalidArgument("Struct does not start with '{' character, found '" +
+                                       ss.str());
+    }
+    if (slice.back() != '}') {
+        std::stringstream ss;
+        ss << slice.back() << '\'';
+        return Status::InvalidArgument("Struct does not end with '}' character, found '" +
+                                       ss.str());
+    }
+
+    Struct struct_field;
+    if (slice.size > 2) {
+        slice = slice.substring(1, slice.size - 2);
+        slice = slice.trim_whitespace();
+        if (!slice.empty()) {
+            FormatOptions options;
+            auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter(slice, [&](char c) {
+                return c == options.map_key_delim || c == options.collection_delim;
+            });
+
+            const auto elem_size = elem_serdes_ptrs.size();
+            std::vector<StringRef> field_values;
+            field_values.reserve(elem_size);
+            if (split_result.size() == elem_size) {
+                for (int i = 0; i < split_result.size(); i++) {
+                    if (i != split_result.size() - 1 &&
+                        split_result[i].delimiter != options.collection_delim) {
+                        return Status::InvalidArgument(
+                                "Struct field value {} is not separated by collection_delim.", i);
+                    }
+                    field_values.push_back(split_result[i].element);
+                }
+            } else if (split_result.size() == 2 * elem_size) {
+                int field_pos = 0;
+                for (int i = 0; i < split_result.size(); i += 2) {
+                    if (split_result[i].delimiter != options.map_key_delim) {
+                        return Status::InvalidArgument(
+                                "Struct name-value pair does not have map key delimiter");
+                    }
+                    if (i != 0 && split_result[i - 1].delimiter != options.collection_delim) {
+                        return Status::InvalidArgument(
+                                "Struct name-value pair does not have collection delimiter");
+                    }
+                    auto field_name = split_result[i].element.trim_quote();
+                    if (!field_name.eq(StringRef(elem_names[field_pos]))) {
+                        return Status::InvalidArgument(
+                                "Cannot find struct field name {} in schema.",
+                                split_result[i].element.to_string());
+                    }
+                    field_values.push_back(split_result[i + 1].element);
+                    field_pos++;
+                }
+            } else {
+                return Status::InvalidArgument(
+                        "Struct field number {} is not equal to schema field number {}.",
+                        split_result.size(), elem_size);
+            }
+
+            struct_field.reserve(elem_size);
+            for (int field_pos = 0; field_pos < elem_size; ++field_pos) {
+                Field elem_field;
+                RETURN_IF_ERROR(ComplexTypeDeserializeUtil::process_field(
+                        elem_serdes_ptrs[field_pos], field_values[field_pos], elem_field));
+                struct_field.push_back(std::move(elem_field));
+            }
+        }
+    }
+
+    field = Field::create_field<TYPE_STRUCT>(std::move(struct_field));
+    return Status::OK();
+}
+
 Status DataTypeStructSerDe::from_string(StringRef& str, IColumn& column,
                                         const FormatOptions& options) const {
     return _from_string<false>(str, column, options);

be/src/core/data_type_serde/data_type_struct_serde.h

@@ -47,6 +47,8 @@ class DataTypeStructSerDe : public DataTypeSerDe {
     Status from_string_strict_mode(StringRef& str, IColumn& column,
                                    const FormatOptions& options) const override;
 
+    Status from_fe_string(const std::string& str, Field& field) const override;
+
     Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
                                       FormatOptions& options) const override;
 

be/src/storage/iterator/vgeneric_iterators.cpp

@@ -48,8 +48,9 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) {
             const auto* column_desc = _schema.column(ordinal);
             auto unique_id = column_desc->unique_id();
             if (_column_iterators_map.count(unique_id) < 1) {
-                RETURN_IF_ERROR(_segment->new_column_iterator(
-                        *column_desc, &_column_iterators_map[unique_id], &opts));
+                RETURN_IF_ERROR(
+                        _segment->new_column_iterator(*column_desc, _schema.data_type(ordinal),
+                                                      &_column_iterators_map[unique_id], &opts));
             }
             _column_iterators.push_back(_column_iterators_map[unique_id].get());
         }

be/src/storage/schema_change/schema_change.cpp

@@ -44,6 +44,7 @@
 #include "core/block/column_with_type_and_name.h"
 #include "core/column/column.h"
 #include "core/column/column_nullable.h"
+#include "core/data_type/data_type_nullable.h"
 #include "exec/common/util.hpp"
 #include "exec/common/variant_util.h"
 #include "exprs/aggregate/aggregate_function.h"
@@ -1528,8 +1529,11 @@ Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping,
                                              const TabletColumn& column_schema,
                                              const std::string& value) {
     if (!column_schema.is_nullable() || value.length() != 0) {
-        RETURN_IF_ERROR(column_schema.get_vec_type()->get_serde()->from_fe_string(
-                value, column_mapping->default_value));
+        auto type = column_schema.get_vec_type();
+        if (value.length() != 0) {
+            type = remove_nullable(type);
+        }
+        RETURN_IF_ERROR(type->get_serde()->from_fe_string(value, column_mapping->default_value));
     }
 
     return Status::OK();