modify be

eldenmoon · Sep 27, 2024 · 7003da0 · 7003da0
1 parent 1de3f22
commit 7003da0
Show file tree

Hide file tree

Showing 23 changed files with 67 additions and 102 deletions.
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
@@ -276,8 +276,7 @@ TabletColumn TabletReader::materialize_column(const TabletColumn& orig) {
                                cast_type.type);
     }
     column_with_cast_type.set_type(filed_type);
-    column_with_cast_type.set_frac(cast_type.scale);
-    column_with_cast_type.set_precision(cast_type.precision);
+    column_with_cast_type.set_precision_frac(cast_type.precision, cast_type.scale);
     return column_with_cast_type;
 }
 

diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
@@ -612,8 +612,10 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
     if (_has_default_value) {
         column->set_default_value(_default_value);
     }
-    if (_is_decimal) {
+    if (_precision >= 0) {
         column->set_precision(_precision);
+    }
+    if (_frac >= 0) {
         column->set_frac(_frac);
     }
     column->set_length(_length);

diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
@@ -24,6 +24,7 @@
 #include <parallel_hashmap/phmap.h>
 
 #include <algorithm>
+#include <cstdint>
 #include <map>
 #include <memory>
 #include <string>
@@ -176,11 +177,11 @@ class TabletColumn {
     const std::vector<TabletColumnPtr>& sparse_columns() const;
     size_t num_sparse_columns() const { return _num_sparse_columns; }
 
-    void set_precision(int32_t precision) {
+    void set_precision_frac(int32_t precision, int32_t frac, bool is_decimal = true) {
         _precision = precision;
-        _is_decimal = true;
+        _frac = frac;
+        _is_decimal = is_decimal;
     }
-    void set_frac(int32_t frac) { _frac = frac; }
 
     Status check_valid() const {
         if (type() != FieldType::OLAP_FIELD_TYPE_ARRAY &&

diff --git a/be/src/vec/columns/column_nothing.h b/be/src/vec/columns/column_nothing.h
@@ -32,6 +32,13 @@ class ColumnNothing final : public COWHelper<IColumnDummy, ColumnNothing> {
 
     ColumnNothing(const ColumnNothing&) = default;
 
+    ColumnPtr permute(const Permutation& perm, size_t limit) const override {
+        return clone_dummy(limit ? std::min(s, limit) : s);
+    }
+    Field operator[](size_t) const override { return {}; }
+    void get(size_t, Field& f) const override { f = {}; }
+    void insert(const Field&) override { ++s; }
+
 public:
     const char* get_family_name() const override { return "Nothing"; }
     MutableColumnPtr clone_dummy(size_t s_) const override { return ColumnNothing::create(s_); }

diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
@@ -346,7 +346,7 @@ void get_field_info_impl(const Field& field, FieldInfo* info) {
 }
 
 void get_base_field_info(const Field& field, FieldInfo* info) {
-    if (field.get_detailed_type_info().type == TypeIndex::Array) {
+    if (field.get_type_id() == TypeIndex::Array) {
         if (field.safe_get<Array>().empty()) {
             info->scalar_type_id = TypeIndex::Nothing;
             ++info->num_dimensions;
@@ -360,24 +360,23 @@ void get_base_field_info(const Field& field, FieldInfo* info) {
     }
 
     // handle scalar types
-    info->scalar_type_id = field.get_detailed_type_info().type;
+    info->scalar_type_id = field.get_type_id();
     info->have_nulls = true;
     info->need_convert = false;
-    info->scale = field.get_detailed_type_info().scale;
-    info->precision = field.get_detailed_type_info().precision;
+    info->scale = field.get_scale();
+    info->precision = field.get_precision();
 
     // Currently the jsonb type should be the top level type, so we should not wrap it in array,
     // see create_array_of_type.
     // TODO we need to support array<jsonb> correctly
-    if (UNLIKELY(field.get_detailed_type_info().type == TypeIndex::JSONB &&
-                 info->num_dimensions > 0)) {
+    if (UNLIKELY(field.get_type_id() == TypeIndex::JSONB && info->num_dimensions > 0)) {
         info->num_dimensions = 0;
         info->need_convert = true;
     }
 }
 
 void get_field_info(const Field& field, FieldInfo* info) {
-    if (field.get_detailed_type_info().type != TypeIndex::Nothing) {
+    if (field.get_type_id() != TypeIndex::Nothing) {
         // Currently we support specify predefined schema for other types include decimal, datetime ...etc
         // so we should set specified info to create correct types, and those predefined types are static and
         // type no need to deduce
@@ -1022,36 +1021,6 @@ void ColumnObject::Subcolumn::get(size_t n, Field& res) const {
                            n);
 }
 
-bool ColumnObject::Subcolumn::get_part_column_type_and_index(const size_t row,
-                                                             ColumnPtr* part_column,
-                                                             DataTypePtr* part_type,
-                                                             size_t* index) const {
-    if (row < num_of_defaults_in_prefix) {
-        return false;
-    }
-    if (is_finalized()) {
-        *part_column = get_finalized_column_ptr();
-        *part_type = get_least_common_type();
-        *index = row;
-        return true;
-    }
-    size_t ind = row;
-    ind -= num_of_defaults_in_prefix;
-    for (size_t i = 0; i < data.size(); ++i) {
-        const auto& part = data[i];
-        const auto& type = data_types[i];
-        if (ind < part->size()) {
-            *part_column = part;
-            *part_type = type;
-            *index = ind;
-            return true;
-        }
-        ind -= part->size();
-    }
-    throw doris::Exception(ErrorCode::OUT_OF_BOUND, "Index ({}) for getting field is out of range",
-                           row);
-}
-
 Field ColumnObject::operator[](size_t n) const {
     Field object;
     get(n, object);

diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
@@ -184,15 +184,6 @@ class ColumnObject final : public COWHelper<IColumn, ColumnObject> {
         void remove_nullable();
 
         void add_new_column_part(DataTypePtr type);
-        // get the column type and index of a specified row
-        // Example row = 10
-        // row 0-7 is column with int type
-        // row 7-8 is column with bigint type
-        // row 9-15 is column with string type
-        // the row 10 is string type and index is 1
-        // Return false if row < num_of_defaults_in_prefix
-        bool get_part_column_type_and_index(const size_t row, ColumnPtr* part_column,
-                                            DataTypePtr* part_type, size_t* index) const;
 
         friend class ColumnObject;
 

diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp
@@ -236,13 +236,11 @@ void get_column_by_type(const vectorized::DataTypePtr& data_type, const std::str
         return;
     }
     if (WhichDataType(*data_type).is_decimal()) {
-        column.set_precision(data_type->get_precision());
-        column.set_frac(data_type->get_scale());
+        column.set_precision_frac(data_type->get_precision(), data_type->get_scale());
         return;
     }
     if (WhichDataType(*data_type).is_date_time_v2()) {
-        column.set_precision(-1);
-        column.set_frac(data_type->get_scale());
+        column.set_precision_frac(-1, data_type->get_scale(), false);
         return;
     }
 }

diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
@@ -340,12 +340,6 @@ class DecimalField {
   * Warning! Prefer to use chunks of columns instead of single values. See Column.h
   */
 
-struct DetailedTypeInfo {
-    TypeIndex type = TypeIndex::Nothing;
-    int scale = -1;
-    int precision = -1;
-};
-
 class Field {
 public:
     struct Types {
@@ -456,12 +450,12 @@ class Field {
       *  since, in its absence, the compiler will still generate the default constructor.
       */
     Field(const Field& rhs) {
-        detailed_type_info = rhs.detailed_type_info;
+        copy_type_info(rhs);
         create(rhs);
     }
 
     Field(Field&& rhs) {
-        detailed_type_info = rhs.detailed_type_info;
+        copy_type_info(rhs);
         create(std::move(rhs));
     }
 
@@ -480,9 +474,15 @@ class Field {
         create(data, size);
     }
 
-    void set_detail_type_info(const DetailedTypeInfo& info) { detailed_type_info = info; }
+    void set_type_info(TypeIndex type, int precision = -1, int scale = -1) {
+        this->type = type;
+        this->precision = precision;
+        this->scale = scale;
+    }
 
-    const DetailedTypeInfo& get_detailed_type_info() const { return detailed_type_info; }
+    int get_precision() const { return precision; }
+    int get_scale() const { return scale; }
+    TypeIndex get_type_id() const { return type; }
 
     void assign_string(const unsigned char* data, size_t size) {
         destroy();
@@ -500,7 +500,7 @@ class Field {
     }
 
     Field& operator=(const Field& rhs) {
-        detailed_type_info = rhs.detailed_type_info;
+        copy_type_info(rhs);
         if (this != &rhs) {
             if (which != rhs.which) {
                 destroy();
@@ -517,7 +517,7 @@ class Field {
     }
 
     Field& operator=(Field&& rhs) {
-        detailed_type_info = rhs.detailed_type_info;
+        copy_type_info(rhs);
         if (this != &rhs) {
             if (which != rhs.which) {
                 destroy();
@@ -727,7 +727,9 @@ class Field {
     Types::Which which;
     // detailed_type_info is used to store the real type of the field, for example, the real type of a Int64 is DateTimeV2
     // or real type of a Decimal32 is Decimal(27, 9)
-    DetailedTypeInfo detailed_type_info;
+    TypeIndex type = TypeIndex::Nothing;
+    int scale = -1;
+    int precision = -1;
 
     /// Assuming there was no allocated state or it was deallocated (see destroy).
     template <typename T>
@@ -784,6 +786,12 @@ class Field {
         which = Types::JSONB;
     }
 
+    void copy_type_info(const Field& rhs) {
+        this->type = rhs.type;
+        this->precision = rhs.precision;
+        this->scale = rhs.scale;
+    }
+
     void create_jsonb(const unsigned char* data, size_t size) {
         new (&storage) JsonbField(reinterpret_cast<const char*>(data), size);
         which = Types::JSONB;

diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
@@ -120,7 +120,7 @@ class IDataType : private boost::noncopyable {
     virtual Field get_type_field(const IColumn& column, int row) const {
         Field field;
         column.get(row, field);
-        field.set_detail_type_info(DetailedTypeInfo {.type = get_type_id()});
+        field.set_type_info(get_type_id());
         return field;
     }
 

diff --git a/be/src/vec/data_types/data_type_array.cpp b/be/src/vec/data_types/data_type_array.cpp
@@ -328,7 +328,7 @@ Field DataTypeArray::get_type_field(const IColumn& column, int row) const {
         res[i] = get_nested_type()->get_type_field(array.get_data(), offset + i);
     }
     Field typed_res(res);
-    typed_res.set_detail_type_info({.type = TypeIndex::Array});
+    typed_res.set_type_info(TypeIndex::Array);
     return typed_res;
 }
 

diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
@@ -241,9 +241,7 @@ class DataTypeDecimal final : public IDataType {
         const auto& decimal_column = static_cast<const ColumnDecimal<T>&>(column);
         Field field;
         decimal_column.get(row, field);
-        field.set_detail_type_info(DetailedTypeInfo {.type = TypeId<T>::value,
-                                                     .scale = static_cast<int>(scale),
-                                                     .precision = static_cast<int>(precision)});
+        field.set_type_info(TypeId<T>::value, static_cast<int>(precision), static_cast<int>(scale));
         return field;
     }
 

diff --git a/be/src/vec/data_types/data_type_jsonb.h b/be/src/vec/data_types/data_type_jsonb.h
@@ -85,7 +85,7 @@ class DataTypeJsonb final : public IDataType {
         const auto& column_data = static_cast<const ColumnString&>(column);
         Field field =
                 JsonbField(column_data.get_data_at(row).data, column_data.get_data_at(row).size);
-        field.set_detail_type_info({.type = TypeIndex::JSONB});
+        field.set_type_info(TypeIndex::JSONB);
         return field;
     }
 

diff --git a/be/src/vec/data_types/data_type_number_base.h b/be/src/vec/data_types/data_type_number_base.h
@@ -168,7 +168,7 @@ class DataTypeNumberBase : public IDataType {
     Field get_type_field(const IColumn& column, int row) const override {
         const auto& column_data = static_cast<const ColumnVector<T>&>(column);
         Field field = column_data.get_data()[row];
-        field.set_detail_type_info(DetailedTypeInfo {.type = get_type_id()});
+        field.set_type_info(get_type_id());
         return field;
     }
 

diff --git a/be/src/vec/data_types/data_type_string.h b/be/src/vec/data_types/data_type_string.h
@@ -97,7 +97,7 @@ class DataTypeString : public IDataType {
     Field get_type_field(const IColumn& column, int row) const override {
         const auto& column_data = static_cast<const ColumnString&>(column);
         Field field(column_data.get_data_at(row).data, column_data.get_data_at(row).size);
-        field.set_detail_type_info({.type = TypeIndex::String});
+        field.set_type_info(TypeIndex::String);
         return field;
     }
 };

diff --git a/be/src/vec/data_types/data_type_time.h b/be/src/vec/data_types/data_type_time.h
@@ -117,8 +117,7 @@ class DataTypeTimeV2 final : public DataTypeNumberBase<Float64> {
     Field get_type_field(const IColumn& column, int row) const override {
         Field field;
         column.get(row, field);
-        field.set_detail_type_info(DetailedTypeInfo {
-                .type = get_type_id(), .scale = static_cast<int>(get_scale()), .precision = 0});
+        field.set_type_info(get_type_id(), 0, static_cast<int>(get_scale()));
         return field;
     }
 

diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h
@@ -172,8 +172,7 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<UInt64> {
                 assert_cast<const ColumnUInt64&, TypeCheckOnRelease::DISABLE>(column);
         Field field;
         column_data.get(row, field);
-        field.set_detail_type_info(DetailedTypeInfo {
-                .type = get_type_id(), .scale = static_cast<int>(get_scale()), .precision = 0});
+        field.set_type_info(get_type_id(), 0, static_cast<int>(get_scale()));
         return field;
     }
 

diff --git a/regression-test/data/variant_p0/predefine/load.out b/regression-test/data/variant_p0/predefine/load.out
@@ -74,11 +74,11 @@ v1.yyy	double	Yes	false	\N	NONE
 
 -- !sql --
 id	bigint	No	true	\N	
-v1	variant<PREDEFINE_COL1:text,PREDEFINE_COL2:int,PREDEFINE_COL3:decimalv3(38,9),PREDEFINE_COL4:datetimev2(0)>	Yes	false	\N	NONE
-v1.PREDEFINE_COL1	text	Yes	false	\N	NONE
-v1.PREDEFINE_COL2	int	Yes	false	\N	NONE
-v1.PREDEFINE_COL3	decimal(38,0)	Yes	false	\N	NONE
-v1.PREDEFINE_COL4	datetime	Yes	false	\N	NONE
+v1	variant	Yes	false	\N	NONE
+v1.PREDEFINE_COL1	smallint	Yes	false	\N	NONE
+v1.PREDEFINE_COL2	double	Yes	false	\N	NONE
+v1.PREDEFINE_COL3	text	Yes	false	\N	NONE
+v1.PREDEFINE_COL4	text	Yes	false	\N	NONE
 v1.predefine_col1	smallint	Yes	false	\N	NONE
 v1.predefine_col2	double	Yes	false	\N	NONE
 v1.predefine_col3	text	Yes	false	\N	NONE
@@ -89,10 +89,10 @@ v1.predefine_col4	text	Yes	false	\N	NONE
 2	{"predefine_col2":1.11111}
 3	{"predefine_col3":"11111.00000"}
 4	{"predefine_col4":"2020-01-01-01"}
-5	{"PREDEFINE_COL1":"1024"}
-6	{"PREDEFINE_COL2":1}
-7	{"PREDEFINE_COL3":"11111.000000000"}
-8	{"PREDEFINE_COL4":"2020-01-01 01:00:00"}
+5	{"PREDEFINE_COL1":1024}
+6	{"PREDEFINE_COL2":1.11111}
+7	{"PREDEFINE_COL3":"11111.00000"}
+8	{"PREDEFINE_COL4":"2020-01-01-01"}
 
 -- !sql --
 1	{"array_boolean":[1,0,1],"array_date":["2021-01-01","2022-01-01","2023-01-01"],"array_datetime":["2021-01-01 00:00:00","2022-01-01 00:00:00","2023-01-01 00:00:00"],"array_datetimev2":["2021-01-01 00:00:00","2022-01-01 00:00:00","2023-01-01 00:00:00"],"array_datev2":["2021-01-01","2022-01-01","2023-01-01"],"array_decimal":["1.100000000","2.200000000","3.300000000"],"array_float":[1.111109972000122],"array_int":[1,2,3],"array_ipv4":["127.0.0.1","172.0.1.1"],"array_ipv6":["ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"],"array_string":["a","b","c"],"boolean_":1,"date_":"2022-01-01","datetime_":"2022-01-01 11:11:11","datetimev2_":"2022-01-01 11:11:11.999999","datev2_":"2022-01-01","decimal_":"188118222.011121920","float_":128.11099243164063,"int_":11111122,"ipv4_":"127.0.0.1","ipv6_":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe","string_":"12111222113","varchar_":"hello world"}
@@ -107,13 +107,13 @@ v1.predefine_col4	text	Yes	false	\N	NONE
 
 -- !sql --
 id	bigint	No	true	\N	
-v1	variant<PREDEFINE_COL1:text,PREDEFINE_COL2:int,PREDEFINE_COL3:decimalv3(38,9),PREDEFINE_COL4:datetimev2(0)>	Yes	false	\N	NONE
+v1	variant	Yes	false	\N	NONE
 v2	variant<dcm:decimalv3(9,0),dt:datetimev2(0)>	Yes	false	\N	NONE
 v3	variant<dcm:decimalv3(9,0),dt:datetimev2(0),ip:ipv6>	Yes	false	\N	NONE
-v1.PREDEFINE_COL1	text	Yes	false	\N	NONE
-v1.PREDEFINE_COL2	int	Yes	false	\N	NONE
-v1.PREDEFINE_COL3	decimal(38,0)	Yes	false	\N	NONE
-v1.PREDEFINE_COL4	datetime	Yes	false	\N	NONE
+v1.PREDEFINE_COL1	smallint	Yes	false	\N	NONE
+v1.PREDEFINE_COL2	double	Yes	false	\N	NONE
+v1.PREDEFINE_COL3	text	Yes	false	\N	NONE
+v1.PREDEFINE_COL4	text	Yes	false	\N	NONE
 v1.a	tinyint	Yes	false	\N	NONE
 v1.predefine_col1	smallint	Yes	false	\N	NONE
 v1.predefine_col2	double	Yes	false	\N	NONE

diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
@@ -22,7 +22,6 @@ suite("variant_mv") {
     sql "set runtime_filter_mode=OFF";
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
     sql "SET enable_agg_state = true"
-    sql "SET use_variant_as_complex_variant = false"
 
     sql """
     drop table if exists github_events1

diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy
@@ -94,7 +94,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){
     }
 
     try {
-        sql "set use_variant_as_complex_variant = false"
         // sparse columns
         def table_name = "sparse_columns"
         create_table table_name