From a07a1341d10d390de8857493b0051e0772014d45 Mon Sep 17 00:00:00 2001 From: Uniqueyou <134280716+wyxxxcat@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:11:31 +0800 Subject: [PATCH] [fix](function) fix error result in auto partition name (#41130) ## Proposed changes Issue Number: close #xxx list partitions support NULL columns as partition columns Before: ``` mysql> select auto_partition_name('list','wyx','null'); +-------------------------------------------------------------+ | auto_partition_name('list','wyx','null') | +-------------------------------------------------------------+ | NULL | +-------------------------------------------------------------+ ``` Now: ``` mysql> select auto_partition_name('list','wyx','null'); +-------------------------------------------------------------+ | auto_partition_name('list','wyx','null') | +-------------------------------------------------------------+ | pwyx3X | +-------------------------------------------------------------+ ``` --- be/src/vec/functions/function_string.h | 59 ++++++++++++------- .../functions/scalar/AutoPartitionName.java | 9 ++- .../nereids_function_p0/scalar_function/A.out | 16 +++++ .../scalar_function/A.groovy | 12 +++- 4 files changed, 69 insertions(+), 27 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 0cb929779d7525..2157db9ed0cd06 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -389,7 +389,7 @@ class FunctionAutoPartitionName : public IFunction { String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 0; } bool is_variadic() const override { return true; } - + bool use_default_implementation_for_nulls() const override { return false; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { return std::make_shared(); } @@ -397,23 +397,36 @@ class FunctionAutoPartitionName : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { size_t argument_size = arguments.size(); - if (argument_size < 2) { - return Status::InvalidArgument( - "auto_partition_name must contains at least two arguments"); - } + auto const_null_map = ColumnUInt8::create(input_rows_count, 0); + auto null_map = ColumnUInt8::create(input_rows_count, 0); std::vector chars_list(argument_size); std::vector offsets_list(argument_size); std::vector is_const_args(argument_size); + std::vector null_list(argument_size); + std::vector argument_null_columns(argument_size); + std::vector argument_columns(argument_size); for (int i = 0; i < argument_size; ++i) { + argument_columns[i] = + block.get_by_position(arguments[i]).column->convert_to_full_column_if_const(); + if (const auto* nullable = + check_and_get_column(*argument_columns[i])) { + null_list[i] = &nullable->get_null_map_data(); + argument_null_columns[i] = nullable->get_null_map_column_ptr(); + argument_columns[i] = nullable->get_nested_column_ptr(); + } else { + null_list[i] = &const_null_map->get_data(); + } + const auto& [col, is_const] = unpack_if_const(block.get_by_position(arguments[i]).column); - const auto* col_str = assert_cast(col.get()); + const auto* col_str = assert_cast(argument_columns[i].get()); chars_list[i] = &col_str->get_chars(); offsets_list[i] = &col_str->get_offsets(); is_const_args[i] = is_const; } + auto res = ColumnString::create(); auto& res_data = res->get_chars(); auto& res_offset = res->get_offsets(); @@ -422,9 +435,9 @@ class FunctionAutoPartitionName : public IFunction { const char* partition_type = chars_list[0]->raw_data(); // partition type is list|range if (std::strncmp(partition_type, "list", 4) == 0) { - return _auto_partition_type_of_list(chars_list, offsets_list, is_const_args, res_data, - res_offset, input_rows_count, argument_size, block, - result, res); + return _auto_partition_type_of_list(chars_list, offsets_list, is_const_args, null_list, + res_data, res_offset, input_rows_count, + argument_size, block, result, res); } else { return _auto_partition_type_of_range(chars_list, offsets_list, is_const_args, res_data, res_offset, input_rows_count, argument_size, block, @@ -472,8 +485,9 @@ class FunctionAutoPartitionName : public IFunction { } Status _auto_partition_type_of_list(std::vector& chars_list, std::vector& offsets_list, - std::vector& is_const_args, auto& res_data, - auto& res_offset, size_t input_rows_count, + std::vector& is_const_args, + const std::vector& null_list, + auto& res_data, auto& res_offset, size_t input_rows_count, size_t argument_size, Block& block, size_t result, auto& res) const { int curr_len = 0; @@ -484,16 +498,21 @@ class FunctionAutoPartitionName : public IFunction { for (int col = 1; col < argument_size; col++) { const auto& current_offsets = *offsets_list[col]; const auto& current_chars = *chars_list[col]; + const auto& current_nullmap = *null_list[col]; - auto idx = index_check_const(row, is_const_args[col]); - int size = current_offsets[idx] - current_offsets[idx - 1]; - const char* raw_chars = - reinterpret_cast(¤t_chars[current_offsets[idx - 1]]); - - // convert string to u16string in order to convert to unicode strings - const std::string raw_str(raw_chars, size); - auto u16string = _string_to_u16string(raw_str); - res_p += _string_to_unicode(u16string) + std::to_string(u16string.size()); + if (current_nullmap[row]) { + res_p += 'X'; + } else { + auto idx = index_check_const(row, is_const_args[col]); + + int size = current_offsets[idx] - current_offsets[idx - 1]; + const char* raw_chars = + reinterpret_cast(¤t_chars[current_offsets[idx - 1]]); + // convert string to u16string in order to convert to unicode strings + const std::string raw_str(raw_chars, size); + auto u16string = _string_to_u16string(raw_str); + res_p += _string_to_unicode(u16string) + std::to_string(u16string.size()); + } } // check the name of length diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java index 716cb0a6ee5127..a69142ddf4776d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AutoPartitionName.java @@ -20,10 +20,9 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; @@ -40,7 +39,7 @@ * GenerateFunction. */ public class AutoPartitionName extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + implements ExplicitlyCastableSignature, AlwaysNotNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT), @@ -69,11 +68,11 @@ public void checkLegalityAfterRewrite() { throw new AnalysisException("function auto_partition_name must contains at least two arguments"); } if (!child(0).isLiteral()) { - throw new AnalysisException("auto_partition_name must accept literal for 1nd argument"); + throw new AnalysisException("auto_partition_name must accept literal for 1st argument"); } final String partition_type = ((VarcharLiteral) getArgument(0)).getStringValue().toLowerCase(); if (!Lists.newArrayList("range", "list").contains(partition_type)) { - throw new AnalysisException("function auto_partition_name must accept range|list for 1nd argument"); + throw new AnalysisException("function auto_partition_name must accept range|list for 1st argument"); } else if (Lists.newArrayList("range").contains(partition_type)) { if (!child(1).isLiteral()) { throw new AnalysisException("auto_partition_name must accept literal for 2nd argument"); diff --git a/regression-test/data/nereids_function_p0/scalar_function/A.out b/regression-test/data/nereids_function_p0/scalar_function/A.out index a08da4e70f4edd..a525070f5bc220 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/A.out +++ b/regression-test/data/nereids_function_p0/scalar_function/A.out @@ -484,6 +484,16 @@ pchar116varchar119 pchar126varchar129 pchar136varchar139 +-- !sql_auto_partition_name_list_column -- +pchar116Xvarchar119 +pchar126Xvarchar129 +pchar136Xvarchar139 + +-- !sql_auto_partition_name_list_column -- +pXstring17XX +pXstring27XX +pXstring37XX + -- !sql_auto_partition_name_list_literal_empty -- p0 @@ -496,6 +506,12 @@ p_2dhello6 -- !sql_auto_partition_name_list_literal_mixed -- p4023ffe5257e7cworld111112e2e2e2e20 +-- !sql_auto_partition_name_list_literal_mixed -- +plist4X11X + +-- !sql_auto_partition_name_list_literal_null -- +pXXX + -- !sql_auto_partition_name_range_literal_notnull -- p20221212000000 diff --git a/regression-test/suites/nereids_function_p0/scalar_function/A.groovy b/regression-test/suites/nereids_function_p0/scalar_function/A.groovy index 8ccdd31102e9de..bc8ffbe71b443d 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/A.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/A.groovy @@ -80,10 +80,14 @@ suite("nereids_scalar_fn_A") { qt_sql_auto_partition_name_list_column_type_mixed "select auto_partition_name('list', kchrs1, kbool) from fn_test_not_nullable where id < 3 order by kchrs1" qt_sql_auto_partition_name_list_column_literal_mixed "select auto_partition_name('list', kstr, 'hello') from fn_test_not_nullable where id < 3 order by kstr" qt_sql_auto_partition_name_list_column "select auto_partition_name('list', kchrs1, kvchrs1) from fn_test_not_nullable where id < 3 order by kchrs1" + qt_sql_auto_partition_name_list_column "select auto_partition_name('list', kchrs1, null, kvchrs1) from fn_test where id < 3 order by kchrs1" + qt_sql_auto_partition_name_list_column "select auto_partition_name('list', null, kstr, null, null) from fn_test where id < 3 order by kstr" qt_sql_auto_partition_name_list_literal_empty "select auto_partition_name('list', '')" qt_sql_auto_partition_name_list_literal_mixed "select auto_partition_name('list', '你好', true, false)" qt_sql_auto_partition_name_list_literal_mixed "select auto_partition_name('list', '-hello')" qt_sql_auto_partition_name_list_literal_mixed "select auto_partition_name('list', '@#￥%~|world11111....')" + qt_sql_auto_partition_name_list_literal_mixed "select auto_partition_name('list', 'list', null, true, null)" + qt_sql_auto_partition_name_list_literal_null "select auto_partition_name('list', null, null, null);" qt_sql_auto_partition_name_range_literal_notnull "select auto_partition_name('range', 'day', '2022-12-12 19:20:30')" qt_sql_auto_partition_name_range_literal_notnull "select auto_partition_name('range', 'month', '2022-12-12 19:20:30')" qt_sql_auto_partition_name_range_literal_notnull "select auto_partition_name('range', 'year', '2022-12-12 19:20:30')" @@ -107,7 +111,7 @@ suite("nereids_scalar_fn_A") { } test{ sql """select auto_partition_name(kdt, 'day', kdt) from fn_test_not_nullable order by kdt""" - exception "auto_partition_name must accept literal for 1nd argument" + exception "auto_partition_name must accept literal for 1st argument" } test{ sql """select auto_partition_name('range', kdt, kdt) from fn_test_not_nullable order by kdt""" @@ -139,7 +143,11 @@ suite("nereids_scalar_fn_A") { } test{ sql """select auto_partition_name('ranges', 'year', 'hello');""" - exception "function auto_partition_name must accept range|list for 1nd argument" + exception "function auto_partition_name must accept range|list for 1st argument" + } + test{ + sql """select auto_partition_name('lists', 'year', 'hello');""" + exception "function auto_partition_name must accept range|list for 1st argument" } test{ sql """select auto_partition_name('range', 'years', 'hello');"""