[Performance](Variant) Improve load performance for variant type

be/src/vec/columns/column_object.cpp

@@ -45,6 +45,7 @@
 #include "util/defer_op.h"
 #include "util/simd/bits.h"
 #include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/helpers.h"
 #include "vec/columns/column.h"
 #include "vec/columns/column_array.h"
 #include "vec/columns/column_nullable.h"
@@ -55,6 +56,7 @@
 #include "vec/common/field_visitors.h"
 #include "vec/common/schema_util.h"
 #include "vec/common/string_buffer.hpp"
+#include "vec/common/string_ref.h"
 #include "vec/core/column_with_type_and_name.h"
 #include "vec/core/field.h"
 #include "vec/core/types.h"
@@ -67,6 +69,7 @@
 #include "vec/data_types/data_type_nothing.h"
 #include "vec/data_types/data_type_nullable.h"
 #include "vec/data_types/get_least_supertype.h"
+#include "vec/json/path_in_data.h"
 
 #ifdef __AVX2__
 #include "util/jsonb_parser_simd.h"
@@ -148,6 +151,84 @@ class FieldVisitorToNumberOfDimensions : public StaticVisitor<size_t> {
     }
 };
 
+// Visitor that allows to get type of scalar field
+// but include contain complex field.This is a faster version
+// for FieldVisitorToScalarType which does not support complex field.
+class SimpleFieldVisitorToScarlarType : public StaticVisitor<size_t> {
+public:
+    size_t operator()(const Array& x) {
+        throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Array type is not supported");
+    }
+    size_t operator()(const UInt64& x) {
+        if (x <= std::numeric_limits<Int8>::max()) {
+            type = TypeIndex::Int8;
+        } else if (x <= std::numeric_limits<Int16>::max()) {
+            type = TypeIndex::Int16;
+        } else if (x <= std::numeric_limits<Int32>::max()) {
+            type = TypeIndex::Int32;
+        } else {
+            type = TypeIndex::Int64;
+        }
+        return 1;
+    }
+    size_t operator()(const Int64& x) {
+        if (x <= std::numeric_limits<Int8>::max() && x >= std::numeric_limits<Int8>::min()) {
+            type = TypeIndex::Int8;
+        } else if (x <= std::numeric_limits<Int16>::max() &&
+                   x >= std::numeric_limits<Int16>::min()) {
+            type = TypeIndex::Int16;
+        } else if (x <= std::numeric_limits<Int32>::max() &&
+                   x >= std::numeric_limits<Int32>::min()) {
+            type = TypeIndex::Int32;
+        } else {
+            type = TypeIndex::Int64;
+        }
+        return 1;
+    }
+    size_t operator()(const JsonbField& x) {
+        type = TypeIndex::JSONB;
+        return 1;
+    }
+    size_t operator()(const Null&) {
+        have_nulls = true;
+        return 1;
+    }
+    template <typename T>
+    size_t operator()(const T&) {
+        type = TypeId<NearestFieldType<T>>::value;
+        return 1;
+    }
+    void get_scalar_type(DataTypePtr* data_type) const {
+        WhichDataType which(type);
+#define DISPATCH(TYPE)                                         \
+    if (which.idx == TypeIndex::TYPE) {                        \
+        *data_type = std::make_shared<DataTypeNumber<TYPE>>(); \
+        return;                                                \
+    }
+        FOR_NUMERIC_TYPES(DISPATCH)
+#undef DISPATCH
+        if (which.is_string()) {
+            *data_type = std::make_shared<DataTypeString>();
+            return;
+        }
+        if (which.is_json()) {
+            *data_type = std::make_shared<DataTypeJsonb>();
+            return;
+        }
+        if (which.is_nothing()) {
+            *data_type = std::make_shared<DataTypeNothing>();
+            return;
+        }
+    }
+    bool contain_nulls() const { return have_nulls; }
+
+    bool need_convert_field() const { return false; }
+
+private:
+    TypeIndex type = TypeIndex::Nothing;
+    bool have_nulls;
+};
+
 /// Visitor that allows to get type of scalar field
 /// or least common type of scalars in array.
 /// More optimized version of FieldToDataType.
@@ -220,8 +301,10 @@ class FieldVisitorToScalarType : public StaticVisitor<size_t> {
 };
 
 } // namespace
-void get_field_info(const Field& field, FieldInfo* info) {
-    FieldVisitorToScalarType to_scalar_type_visitor;
+
+template <typename Visitor>
+void get_field_info_impl(const Field& field, FieldInfo* info) {
+    Visitor to_scalar_type_visitor;
     apply_visitor(to_scalar_type_visitor, field);
     DataTypePtr type = nullptr;
     to_scalar_type_visitor.get_scalar_type(&type);
@@ -234,6 +317,14 @@ void get_field_info(const Field& field, FieldInfo* info) {
     };
 }
 
+void get_field_info(const Field& field, FieldInfo* info) {
+    if (field.is_complex_field()) {
+        get_field_info_impl<FieldVisitorToScalarType>(field, info);
+    } else {
+        get_field_info_impl<SimpleFieldVisitorToScarlarType>(field, info);
+    }
+}
+
 ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr&& data_, DataTypePtr type, bool is_nullable_,
                                    bool is_root_)
         : least_common_type(type), is_nullable(is_nullable_), is_root(is_root_) {
@@ -316,8 +407,8 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) {
     if (data.empty()) {
         add_new_column_part(create_array_of_type(std::move(base_type), value_dim, is_nullable));
     } else if (!least_common_base_type->equals(*base_type) && !is_nothing(base_type)) {
-        if (!schema_util::is_conversion_required_between_integers(*base_type,
-                                                                  *least_common_base_type)) {
+        if (schema_util::is_conversion_required_between_integers(*base_type,
+                                                                 *least_common_base_type)) {
             get_least_supertype<LeastSupertypeOnError::Jsonb>(
                     DataTypes {std::move(base_type), least_common_base_type}, &base_type);
             type_changed = true;
@@ -676,14 +767,12 @@ void ColumnObject::try_insert(const Field& field) {
         return;
     }
     const auto& object = field.get<const VariantMap&>();
-    phmap::flat_hash_set<std::string> inserted;
     size_t old_size = size();
     for (const auto& [key_str, value] : object) {
         PathInData key;
         if (!key_str.empty()) {
             key = PathInData(key_str);
         }
-        inserted.insert(key_str);
         if (!has_subcolumn(key)) {
             bool succ = add_sub_column(key, old_size);
             if (!succ) {
@@ -699,7 +788,7 @@ void ColumnObject::try_insert(const Field& field) {
         subcolumn->insert(value);
     }
     for (auto& entry : subcolumns) {
-        if (!inserted.contains(entry->path.get_path())) {
+        if (old_size == entry->data.size()) {
             entry->data.insertDefault();
         }
     }

be/src/vec/columns/column_object.h

@@ -72,6 +72,7 @@ struct FieldInfo {
     /// Number of dimension in array. 0 if field is scalar.
     size_t num_dimensions;
 };
+
 void get_field_info(const Field& field, FieldInfo* info);
 /** A column that represents object with dynamic set of subcolumns.
  *  Subcolumns are identified by paths in document and are stored in

be/src/vec/core/field.h

@@ -482,6 +482,11 @@ class Field {
         return *this;
     }
 
+    bool is_complex_field() const {
+        return which == Types::Array || which == Types::Map || which == Types::Tuple ||
+               which == Types::VariantMap;
+    }
+
     Field& operator=(Field&& rhs) {
         if (this != &rhs) {
             if (which != rhs.which) {

be/src/vec/json/parse2column.cpp

@@ -148,36 +148,28 @@ void parse_json_to_variant(IColumn& column, const char* src, size_t length,
     }
     auto& [paths, values] = *result;
     assert(paths.size() == values.size());
-    phmap::flat_hash_set<std::string> paths_set;
-    size_t num_rows = column_object.size();
+    size_t old_num_rows = column_object.size();
     for (size_t i = 0; i < paths.size(); ++i) {
         FieldInfo field_info;
         get_field_info(values[i], &field_info);
         if (is_nothing(field_info.scalar_type)) {
             continue;
         }
-        if (!paths_set.insert(paths[i].get_path()).second) {
-            // return Status::DataQualityError(
-            //         fmt::format("Object has ambiguous path {}, {}", paths[i].get_path()));
-            throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Object has ambiguous path {}",
-                                   paths[i].get_path());
-        }
-
         if (!column_object.has_subcolumn(paths[i])) {
-            column_object.add_sub_column(paths[i], num_rows);
+            column_object.add_sub_column(paths[i], old_num_rows);
         }
         auto* subcolumn = column_object.get_subcolumn(paths[i]);
         if (!subcolumn) {
             throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Failed to find sub column {}",
                                    paths[i].get_path());
         }
-        assert(subcolumn->size() == num_rows);
+        assert(subcolumn->size() == old_num_rows);
         subcolumn->insert(std::move(values[i]), std::move(field_info));
     }
     // /// Insert default values to missed subcolumns.
     const auto& subcolumns = column_object.get_subcolumns();
     for (const auto& entry : subcolumns) {
-        if (!paths_set.contains(entry->path.get_path())) {
+        if (entry->data.size() == old_num_rows) {
             entry->data.insertDefault();
         }
     }

regression-test/suites/variant_p2/performance.groovy

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("regression_test_variant_performance", "p2"){
+    sql """CREATE TABLE IF NOT EXISTS var_perf (
+                        k bigint,
+                        v variant
+
+                    )
+                    DUPLICATE KEY(`k`)
+                    DISTRIBUTED BY RANDOM BUCKETS 4
+                    properties("replication_num" = "1", "disable_auto_compaction" = "false");
+    """
+    sql """
+        insert into var_perf 
+                SELECT *, '{"field1":348,"field2":596,"field3":781,"field4":41,"field5":922,"field6":84,"field7":222,"field8":312,"field9":490,"field10":715,"field11":837,"field12":753,"field13":171,"field14":727,"field15":739,"field16":545,"field17":964,"field18":540,"field19":685,"field20":828,"field21":157,"field22":404,"field23":287,"field24":481,"field25":476,"field26":559,"field27":144,"field28":545,"field29":70,"field30":668,"field31":820,"field32":193,"field33":465,"field34":347,"field35":898,"field36":705,"field37":754,"field38":866,"field39":752,"field40":303,"field41":214,"field42":41,"field43":609,"field44":487,"field45":832,"field46":832,"field47":134,"field48":964,"field49":919,"field50":670,"field51":767,"field52":334,"field53":506,"field54":838,"field55":510,"field56":770,"field57":168,"field58":701,"field59":961,"field60":927,"field61":375,"field62":939,"field63":464,"field64":420,"field65":212,"field66":882,"field67":344,"field68":724,"field69":997,"field70":198,"field71":739,"field72":628,"field73":563,"field74":979,"field75":563,"field76":891,"field77":496,"field78":442,"field79":847,"field80":771,"field81":229,"field82":1023,"field83":184,"field84":563,"field85":980,"field86":191,"field87":426,"field88":527,"field89":945,"field90":552,"field91":454,"field92":728,"field93":631,"field94":191,"field95":148,"field96":679,"field97":955,"field98":934,"field99":258,"field100":442}'
+                        from numbers("number" = "10000000")
+                union all
+                SELECT *, '{"field1":201,"field2":465,"field3":977,"field4":101112,"field5":131415,"field6":216,"field7":192021,"field8":822324,"field9":525627,"field10":928930,"field11":413233,"field12":243536,"field13":373839,"field14":404142,"field15":434445,"field16":1464748,"field17":495051,"field18":525354,"field19":565657,"field20":1585960,"field21":616263,"field22":646566,"field23":676869,"field24":707172,"field25":737475,"field26":767778,"field27":798081,"field28":828384,"field29":858687,"field30":888990,"field31":919293,"field32":949596,"field33":979899,"field34":100101,"field35":103104,"field36":106107,"field37":109110,"field38":112113,"field39":115116,"field40":118119,"field41":121122,"field42":124125,"field43":127128,"field44":130131,"field45":133134,"field46":136137,"field47":139140,"field48":142143,"field49":145146,"field50":148149,"field51":151152,"field52":154155,"field53":157158,"field54":160161,"field55":163164,"field56":166167,"field57":169170,"field58":172173,"field59":175176,"field60":178179,"field61":181182,"field62":184185,"field63":187188,"field64":190191,"field65":193194,"field66":196197,"field67":199200,"field68":202203,"field69":205206,"field70":208209,"field71":211212,"field72":214215,"field73":217218,"field74":220221,"field75":223224,"field76":226227,"field77":229230,"field78":232233,"field79":235236,"field80":238239,"field81":241242,"field82":244245,"field83":247248,"field84":250251,"field85":253254,"field86":256257,"field87":259260,"field88":262263,"field89":265266,"field90":268269,"field91":271272,"field92":274275,"field93":277278,"field94":280281,"field95":283284,"field96":286287,"field97":289290,"field98":292293,"field99":295296,"field100":298299}'
+                        from numbers("number" = "10000000")
+        """
+}