diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp index b4d993ef035acb..9256d1deb2b072 100644 --- a/be/src/pipeline/exec/table_function_operator.cpp +++ b/be/src/pipeline/exec/table_function_operator.cpp @@ -53,13 +53,15 @@ Status TableFunctionLocalState::open(RuntimeState* state) { for (size_t i = 0; i < _vfn_ctxs.size(); i++) { RETURN_IF_ERROR(p._vfn_ctxs[i]->clone(state, _vfn_ctxs[i])); - const std::string& tf_name = _vfn_ctxs[i]->root()->fn().name.function_name; vectorized::TableFunction* fn = nullptr; - RETURN_IF_ERROR(vectorized::TableFunctionFactory::get_fn(tf_name, state->obj_pool(), &fn)); + RETURN_IF_ERROR(vectorized::TableFunctionFactory::get_fn(_vfn_ctxs[i]->root()->fn(), + state->obj_pool(), &fn)); fn->set_expr_context(_vfn_ctxs[i]); _fns.push_back(fn); } - + for (auto* fn : _fns) { + RETURN_IF_ERROR(fn->open()); + } _cur_child_offset = -1; return Status::OK(); } @@ -138,6 +140,7 @@ bool TableFunctionLocalState::_roll_table_functions(int last_eos_idx) { bool TableFunctionLocalState::_is_inner_and_empty() { for (int i = 0; i < _parent->cast()._fn_num; i++) { // if any table function is not outer and has empty result, go to next child row + // if it's outer function, will be insert into one row NULL if (!_fns[i]->is_outer() && _fns[i]->current_empty()) { return true; } @@ -269,9 +272,8 @@ Status TableFunctionOperatorX::init(const TPlanNode& tnode, RuntimeState* state) _vfn_ctxs.push_back(ctx); auto root = ctx->root(); - const std::string& tf_name = root->fn().name.function_name; vectorized::TableFunction* fn = nullptr; - RETURN_IF_ERROR(vectorized::TableFunctionFactory::get_fn(tf_name, _pool, &fn)); + RETURN_IF_ERROR(vectorized::TableFunctionFactory::get_fn(root->fn(), _pool, &fn)); fn->set_expr_context(ctx); _fns.push_back(fn); } diff --git a/be/src/pipeline/exec/table_function_operator.h b/be/src/pipeline/exec/table_function_operator.h index 49dd242bfe78d9..8a7b7bd43d45d1 100644 --- a/be/src/pipeline/exec/table_function_operator.h +++ b/be/src/pipeline/exec/table_function_operator.h @@ -56,6 +56,13 @@ class TableFunctionLocalState final : public PipelineXLocalState<> { ~TableFunctionLocalState() override = default; Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override { + for (auto* fn : _fns) { + RETURN_IF_ERROR(fn->close()); + } + RETURN_IF_ERROR(PipelineXLocalState<>::close(state)); + return Status::OK(); + } void process_next_child_row(); Status get_expanded_block(RuntimeState* state, vectorized::Block* output_block, bool* eos); @@ -74,7 +81,7 @@ class TableFunctionLocalState final : public PipelineXLocalState<> { std::vector _fns; vectorized::VExprContextSPtrs _vfn_ctxs; - int64_t _cur_child_offset = 0; + int64_t _cur_child_offset = -1; std::unique_ptr _child_block; int _current_row_insert_times = 0; bool _child_eos = false; diff --git a/be/src/vec/columns/column_array.cpp b/be/src/vec/columns/column_array.cpp index 591eb74ca09178..aba07557e830e0 100644 --- a/be/src/vec/columns/column_array.cpp +++ b/be/src/vec/columns/column_array.cpp @@ -416,7 +416,7 @@ void ColumnArray::insert(const Field& x) { } void ColumnArray::insert_from(const IColumn& src_, size_t n) { - DCHECK(n < src_.size()); + DCHECK_LT(n, src_.size()); const ColumnArray& src = assert_cast(src_); size_t size = src.size_at(n); size_t offset = src.offset_at(n); diff --git a/be/src/vec/exec/vtable_function_node.cpp b/be/src/vec/exec/vtable_function_node.cpp index 0c35fae806ea93..8a04be3b6bb4b0 100644 --- a/be/src/vec/exec/vtable_function_node.cpp +++ b/be/src/vec/exec/vtable_function_node.cpp @@ -58,9 +58,8 @@ Status VTableFunctionNode::init(const TPlanNode& tnode, RuntimeState* state) { _vfn_ctxs.push_back(ctx); auto root = ctx->root(); - const std::string& tf_name = root->fn().name.function_name; TableFunction* fn = nullptr; - RETURN_IF_ERROR(TableFunctionFactory::get_fn(tf_name, _pool, &fn)); + RETURN_IF_ERROR(TableFunctionFactory::get_fn(root->fn(), _pool, &fn)); fn->set_expr_context(ctx); _fns.push_back(fn); } @@ -93,6 +92,7 @@ Status VTableFunctionNode::_prepare_output_slot_ids(const TPlanNode& tnode) { bool VTableFunctionNode::_is_inner_and_empty() { for (int i = 0; i < _fn_num; i++) { // if any table function is not outer and has empty result, go to next child row + // if it's outer function, will be insert into NULL if (!_fns[i]->is_outer() && _fns[i]->current_empty()) { return true; } diff --git a/be/src/vec/exec/vtable_function_node.h b/be/src/vec/exec/vtable_function_node.h index 0b64fe47cc541f..41dbd8bab64230 100644 --- a/be/src/vec/exec/vtable_function_node.h +++ b/be/src/vec/exec/vtable_function_node.h @@ -58,7 +58,11 @@ class VTableFunctionNode final : public ExecNode { Status alloc_resource(RuntimeState* state) override { SCOPED_TIMER(_exec_timer); RETURN_IF_ERROR(ExecNode::alloc_resource(state)); - return VExpr::open(_vfn_ctxs, state); + RETURN_IF_ERROR(VExpr::open(_vfn_ctxs, state)); + for (auto* fn : _fns) { + RETURN_IF_ERROR(fn->open()); + } + return Status::OK(); } Status get_next(RuntimeState* state, Block* block, bool* eos) override; bool need_more_input_data() const { return !_child_block->rows() && !_child_eos; } @@ -67,6 +71,9 @@ class VTableFunctionNode final : public ExecNode { if (_num_rows_filtered_counter != nullptr) { COUNTER_SET(_num_rows_filtered_counter, static_cast(_num_rows_filtered)); } + for (auto* fn : _fns) { + static_cast(fn->close()); + } ExecNode::release_resource(state); } @@ -145,7 +152,7 @@ class VTableFunctionNode final : public ExecNode { std::shared_ptr _child_block; std::vector _child_slots; std::vector _output_slots; - int64_t _cur_child_offset = 0; + int64_t _cur_child_offset = -1; VExprContextSPtrs _vfn_ctxs; diff --git a/be/src/vec/exprs/table_function/table_function.h b/be/src/vec/exprs/table_function/table_function.h index 98d811364399f5..c817067470a22a 100644 --- a/be/src/vec/exprs/table_function/table_function.h +++ b/be/src/vec/exprs/table_function/table_function.h @@ -58,6 +58,7 @@ class TableFunction { virtual int get_value(MutableColumnPtr& column, int max_step) { max_step = std::max(1, std::min(max_step, (int)(_cur_size - _cur_offset))); int i = 0; + // TODO: this for loop maybe could refactor, and call once get_value function, it's could insert into max_step value once for (; i < max_step && !eos(); i++) { get_value(column); forward(); diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index e42c0a27fd1111..29b201b59479d5 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -17,10 +17,14 @@ #include "vec/exprs/table_function/table_function_factory.h" +#include + +#include #include #include "common/object_pool.h" #include "vec/exprs/table_function/table_function.h" +#include "vec/exprs/table_function/udf_table_function.h" #include "vec/exprs/table_function/vexplode.h" #include "vec/exprs/table_function/vexplode_bitmap.h" #include "vec/exprs/table_function/vexplode_json_array.h" @@ -65,23 +69,30 @@ const std::unordered_map {}}, {"explode", TableFunctionCreator {}}}; -Status TableFunctionFactory::get_fn(const std::string& fn_name_raw, ObjectPool* pool, - TableFunction** fn) { - bool is_outer = match_suffix(fn_name_raw, COMBINATOR_SUFFIX_OUTER); - std::string fn_name_real = - is_outer ? remove_suffix(fn_name_raw, COMBINATOR_SUFFIX_OUTER) : fn_name_raw; - - auto fn_iterator = _function_map.find(fn_name_real); - if (fn_iterator != _function_map.end()) { - *fn = pool->add(fn_iterator->second().release()); +Status TableFunctionFactory::get_fn(const TFunction& t_fn, ObjectPool* pool, TableFunction** fn) { + bool is_outer = match_suffix(t_fn.name.function_name, COMBINATOR_SUFFIX_OUTER); + if (t_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { + *fn = pool->add(UDFTableFunction::create_unique(t_fn).release()); if (is_outer) { (*fn)->set_outer(); } - return Status::OK(); - } + } else { + const std::string& fn_name_raw = t_fn.name.function_name; + const std::string& fn_name_real = + is_outer ? remove_suffix(fn_name_raw, COMBINATOR_SUFFIX_OUTER) : fn_name_raw; - return Status::NotSupported("Table function {} is not support", fn_name_raw); + auto fn_iterator = _function_map.find(fn_name_real); + if (fn_iterator != _function_map.end()) { + *fn = pool->add(fn_iterator->second().release()); + if (is_outer) { + (*fn)->set_outer(); + } + + return Status::OK(); + } + } + return Status::NotSupported("Table function {} is not support", t_fn.name.function_name); } } // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/table_function_factory.h b/be/src/vec/exprs/table_function/table_function_factory.h index a68a1763fc44e9..cd06c202f3778c 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.h +++ b/be/src/vec/exprs/table_function/table_function_factory.h @@ -17,6 +17,8 @@ #pragma once +#include + #include #include #include @@ -33,7 +35,7 @@ class TableFunction; class TableFunctionFactory { public: TableFunctionFactory() = delete; - static Status get_fn(const std::string& fn_name_raw, ObjectPool* pool, TableFunction** fn); + static Status get_fn(const TFunction& t_fn, ObjectPool* pool, TableFunction** fn); const static std::unordered_map()>> _function_map; diff --git a/be/src/vec/exprs/table_function/udf_table_function.cpp b/be/src/vec/exprs/table_function/udf_table_function.cpp new file mode 100644 index 00000000000000..bc4c815ceb15cd --- /dev/null +++ b/be/src/vec/exprs/table_function/udf_table_function.cpp @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exprs/table_function/udf_table_function.h" + +#include + +#include "runtime/user_function_cache.h" +#include "vec/columns/column_array.h" +#include "vec/columns/column_nullable.h" +#include "vec/common/assert_cast.h" +#include "vec/core/block.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_factory.hpp" +#include "vec/exec/jni_connector.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" + +namespace doris::vectorized { +const char* EXECUTOR_CLASS = "org/apache/doris/udf/UdfExecutor"; +const char* EXECUTOR_CTOR_SIGNATURE = "([B)V"; +const char* EXECUTOR_EVALUATE_SIGNATURE = "(Ljava/util/Map;Ljava/util/Map;)J"; +const char* EXECUTOR_CLOSE_SIGNATURE = "()V"; +UDFTableFunction::UDFTableFunction(const TFunction& t_fn) : TableFunction(), _t_fn(t_fn) { + _fn_name = _t_fn.name.function_name; + _return_type = DataTypeFactory::instance().create_data_type( + TypeDescriptor::from_thrift(t_fn.ret_type)); + // as the java-utdf function in java code is eg: ArrayList + // so we need a array column to save the execute result, and make_nullable could help deal with nullmap + _return_type = make_nullable(std::make_shared(make_nullable(_return_type))); +} + +Status UDFTableFunction::open() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + if (env == nullptr) { + return Status::InternalError("Failed to get/create JVM"); + } + _jni_ctx = std::make_shared(); + // Add a scoped cleanup jni reference object. This cleans up local refs made below. + JniLocalFrame jni_frame; + { + std::string local_location; + auto* function_cache = UserFunctionCache::instance(); + RETURN_IF_ERROR(function_cache->get_jarpath(_t_fn.id, _t_fn.hdfs_location, _t_fn.checksum, + &local_location)); + TJavaUdfExecutorCtorParams ctor_params; + ctor_params.__set_fn(_t_fn); + ctor_params.__set_location(local_location); + jbyteArray ctor_params_bytes; + // Pushed frame will be popped when jni_frame goes out-of-scope. + RETURN_IF_ERROR(jni_frame.push(env)); + RETURN_IF_ERROR(SerializeThriftMsg(env, &ctor_params, &ctor_params_bytes)); + RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, EXECUTOR_CLASS, &_jni_ctx->executor_cl)); + _jni_ctx->executor_ctor_id = + env->GetMethodID(_jni_ctx->executor_cl, "", EXECUTOR_CTOR_SIGNATURE); + _jni_ctx->executor_evaluate_id = + env->GetMethodID(_jni_ctx->executor_cl, "evaluate", EXECUTOR_EVALUATE_SIGNATURE); + _jni_ctx->executor_close_id = + env->GetMethodID(_jni_ctx->executor_cl, "close", EXECUTOR_CLOSE_SIGNATURE); + _jni_ctx->executor = env->NewObject(_jni_ctx->executor_cl, _jni_ctx->executor_ctor_id, + ctor_params_bytes); + jbyte* pBytes = env->GetByteArrayElements(ctor_params_bytes, nullptr); + env->ReleaseByteArrayElements(ctor_params_bytes, pBytes, JNI_ABORT); + env->DeleteLocalRef(ctor_params_bytes); + } + RETURN_ERROR_IF_EXC(env); + RETURN_IF_ERROR(JniUtil::LocalToGlobalRef(env, _jni_ctx->executor, &_jni_ctx->executor)); + _jni_ctx->open_successes = true; + return Status::OK(); +} + +Status UDFTableFunction::process_init(Block* block, RuntimeState* state) { + auto child_size = _expr_context->root()->children().size(); + std::vector child_column_idxs; + child_column_idxs.resize(child_size); + for (int i = 0; i < child_size; ++i) { + int result_id = -1; + RETURN_IF_ERROR(_expr_context->root()->children()[i]->execute(_expr_context.get(), block, + &result_id)); + DCHECK_NE(result_id, -1); + child_column_idxs[i] = result_id; + } + JNIEnv* env = nullptr; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + std::unique_ptr input_table; + RETURN_IF_ERROR( + JniConnector::to_java_table(block, block->rows(), child_column_idxs, input_table)); + auto input_table_schema = JniConnector::parse_table_schema(block, child_column_idxs, true); + std::map input_params = { + {"meta_address", std::to_string((long)input_table.get())}, + {"required_fields", input_table_schema.first}, + {"columns_types", input_table_schema.second}}; + + jobject input_map = JniUtil::convert_to_java_map(env, input_params); + _array_result_column = _return_type->create_column(); + _result_column_idx = block->columns(); + block->insert({_array_result_column, _return_type, "res"}); + auto output_table_schema = JniConnector::parse_table_schema(block, {_result_column_idx}, true); + std::string output_nullable = _return_type->is_nullable() ? "true" : "false"; + std::map output_params = {{"is_nullable", output_nullable}, + {"required_fields", output_table_schema.first}, + {"columns_types", output_table_schema.second}}; + + jobject output_map = JniUtil::convert_to_java_map(env, output_params); + DCHECK(_jni_ctx != nullptr); + DCHECK(_jni_ctx->executor != nullptr); + long output_address = env->CallLongMethod(_jni_ctx->executor, _jni_ctx->executor_evaluate_id, + input_map, output_map); + RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); + env->DeleteLocalRef(input_map); + env->DeleteLocalRef(output_map); + RETURN_IF_ERROR(JniConnector::fill_block(block, {_result_column_idx}, output_address)); + block->erase(_result_column_idx); + if (!extract_column_array_info(*_array_result_column, _array_column_detail)) { + return Status::NotSupported("column type {} not supported now", + block->get_by_position(_result_column_idx).column->get_name()); + } + return Status::OK(); +} + +void UDFTableFunction::process_row(size_t row_idx) { + TableFunction::process_row(row_idx); + if (!_array_column_detail.array_nullmap_data || + !_array_column_detail.array_nullmap_data[row_idx]) { + _array_offset = (*_array_column_detail.offsets_ptr)[row_idx - 1]; + _cur_size = (*_array_column_detail.offsets_ptr)[row_idx] - _array_offset; + } + // so when it's NULL of row_idx, will not update _cur_size + // it's will be _cur_size == 0, and means current_empty. + // if the fn is outer, will be continue insert_default + // if the fn is not outer function, will be not insert any value. +} + +void UDFTableFunction::process_close() { + _array_result_column = nullptr; + _array_column_detail.reset(); + _array_offset = 0; +} + +void UDFTableFunction::get_value(MutableColumnPtr& column) { + size_t pos = _array_offset + _cur_offset; + if (current_empty() || (_array_column_detail.nested_nullmap_data && + _array_column_detail.nested_nullmap_data[pos])) { + column->insert_default(); + } else { + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + auto nested_column = nullable_column->get_nested_column_ptr(); + auto nullmap_column = nullable_column->get_null_map_column_ptr(); + nested_column->insert_from(*_array_column_detail.nested_col, pos); + assert_cast(nullmap_column.get())->insert_default(); + } else { + column->insert_from(*_array_column_detail.nested_col, pos); + } + } +} + +int UDFTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + size_t pos = _array_offset + _cur_offset; + if (current_empty()) { + column->insert_default(); + max_step = 1; + } else { + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + auto nested_column = nullable_column->get_nested_column_ptr(); + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + nested_column->insert_range_from(*_array_column_detail.nested_col, pos, max_step); + size_t old_size = nullmap_column->size(); + nullmap_column->resize(old_size + max_step); + memcpy(nullmap_column->get_data().data() + old_size, + _array_column_detail.nested_nullmap_data + pos * sizeof(UInt8), + max_step * sizeof(UInt8)); + } else { + column->insert_range_from(*_array_column_detail.nested_col, pos, max_step); + } + } + forward(max_step); + return max_step; +} +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/udf_table_function.h b/be/src/vec/exprs/table_function/udf_table_function.h new file mode 100644 index 00000000000000..ae6a7c13b355a9 --- /dev/null +++ b/be/src/vec/exprs/table_function/udf_table_function.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "jni.h" +#include "util/jni-util.h" +#include "vec/columns/column.h" +#include "vec/data_types/data_type.h" +#include "vec/exprs/table_function/table_function.h" +#include "vec/functions/array/function_array_utils.h" + +namespace doris::vectorized { + +class UDFTableFunction final : public TableFunction { + ENABLE_FACTORY_CREATOR(UDFTableFunction); + +public: + UDFTableFunction(const TFunction& t_fn); + ~UDFTableFunction() override = default; + + Status open() override; + Status process_init(Block* block, RuntimeState* state) override; + void process_row(size_t row_idx) override; + void process_close() override; + void get_value(MutableColumnPtr& column) override; + int get_value(MutableColumnPtr& column, int max_step) override; + Status close() override { + if (_jni_ctx) { + RETURN_IF_ERROR(_jni_ctx->close()); + } + return TableFunction::close(); + } + +private: + struct JniContext { + // Do not save parent directly, because parent is in VExpr, but jni context is in FunctionContext + // The deconstruct sequence is not determined, it will core. + // JniContext's lifecycle should same with function context, not related with expr + jclass executor_cl; + jmethodID executor_ctor_id; + jmethodID executor_evaluate_id; + jmethodID executor_close_id; + jobject executor = nullptr; + bool is_closed = false; + bool open_successes = false; + + JniContext() = default; + + Status close() { + if (!open_successes) { + LOG_WARNING("maybe open failed, need check the reason"); + return Status::OK(); //maybe open failed, so can't call some jni + } + if (is_closed) { + return Status::OK(); + } + JNIEnv* env = nullptr; + Status status = JniUtil::GetJNIEnv(&env); + if (!status.ok() || env == nullptr) { + LOG(WARNING) << "errors while get jni env " << status; + return status; + } + env->CallNonvirtualVoidMethodA(executor, executor_cl, executor_close_id, nullptr); + env->DeleteGlobalRef(executor); + env->DeleteGlobalRef(executor_cl); + RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); + is_closed = true; + return Status::OK(); + } + }; + + const TFunction& _t_fn; + std::shared_ptr _jni_ctx = nullptr; + DataTypePtr _return_type = nullptr; + ColumnPtr _array_result_column = nullptr; + ColumnArrayExecutionData _array_column_detail; + size_t _result_column_idx = 0; // _array_result_column pos in block + size_t _array_offset = 0; // start offset of array[row_idx] +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index 1c08b721cf9089..b84499653ebf9b 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -39,6 +39,7 @@ #include "vec/data_types/data_type_agg_state.h" #include "vec/exprs/vexpr_context.h" #include "vec/functions/function_agg_state.h" +#include "vec/functions/function_fake.h" #include "vec/functions/function_java_udf.h" #include "vec/functions/function_rpc.h" #include "vec/functions/simple_function_factory.h" @@ -67,12 +68,16 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, _expr_name = fmt::format("VectorizedFnCall[{}](arguments={},return={})", _fn.name.function_name, get_child_names(), _data_type->get_name()); - if (_fn.binary_type == TFunctionBinaryType::RPC) { _function = FunctionRPC::create(_fn, argument_template, _data_type); } else if (_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { if (config::enable_java_support) { - _function = JavaFunctionCall::create(_fn, argument_template, _data_type); + if (_fn.is_udtf_function) { + // fake function. it's no use and can't execute. + _function = FunctionFake::create(); + } else { + _function = JavaFunctionCall::create(_fn, argument_template, _data_type); + } } else { return Status::InternalError( "Java UDF is not enabled, you can change be config enable_java_support to true " diff --git a/be/src/vec/functions/function_fake.h b/be/src/vec/functions/function_fake.h index 0dabdfb3c83f82..b4891ed9392576 100644 --- a/be/src/vec/functions/function_fake.h +++ b/be/src/vec/functions/function_fake.h @@ -63,4 +63,12 @@ class FunctionFake : public IFunction { } }; +struct UDTFImpl { + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + DCHECK(false) << "get_return_type_impl not supported, shouldn't into here."; + return nullptr; + } + static std::string get_error_msg() { return "Fake function do not support execute"; } +}; + } // namespace doris::vectorized diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index 8ad171d60138f1..2cb8ed5351fd37 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -17,6 +17,7 @@ package org.apache.doris.udf; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Type; import org.apache.doris.common.exception.InternalException; import org.apache.doris.common.exception.UdfRuntimeException; @@ -88,6 +89,9 @@ public BaseExecutor(byte[] thriftParams) throws Exception { fn = request.fn; String jarFile = request.location; Type funcRetType = Type.fromThrift(request.fn.ret_type); + if (request.fn.is_udtf_function) { + funcRetType = ArrayType.create(funcRetType, true); + } init(request, jarFile, funcRetType, parameterTypes); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 00b40b9fc9f35f..52c7e85239b441 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -78,9 +78,11 @@ public final class FeMetaVersion { public static final int VERSION_128 = 128; // For table version public static final int VERSION_129 = 129; + // for java-udtf add a bool field to write + public static final int VERSION_131 = 131; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_129; + public static final int VERSION_CURRENT = VERSION_131; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index a0564670bb23ef..4a20af83f5a53c 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1797,6 +1797,11 @@ create_stmt ::= {: RESULT = new CreateFunctionStmt(type, ifNotExists, functionName, args, parameters, func); :} + | KW_CREATE opt_var_type:type KW_TABLES KW_FUNCTION opt_if_not_exists:ifNotExists function_name:functionName LPAREN func_args_def:args RPAREN + KW_RETURNS type_def:returnType opt_intermediate_type:intermediateType opt_properties:properties + {: + RESULT = new CreateFunctionStmt(type, ifNotExists, functionName, args, returnType, intermediateType, properties); + :} /* Table */ | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name KW_LIKE table_name:existed_name KW_WITH KW_ROLLUP LPAREN ident_list:rollupNames RPAREN {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java index d498d1f75bc665..ed618d1603d669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java @@ -104,6 +104,7 @@ public class CreateFunctionStmt extends DdlStmt { private final FunctionName functionName; private final boolean isAggregate; private final boolean isAlias; + private boolean isTableFunction; private final FunctionArgsDef argsDef; private final TypeDef returnType; private TypeDef intermediateType; @@ -140,10 +141,18 @@ public CreateFunctionStmt(SetType type, boolean ifNotExists, boolean isAggregate this.properties = ImmutableSortedMap.copyOf(properties, String.CASE_INSENSITIVE_ORDER); } this.isAlias = false; + this.isTableFunction = false; this.parameters = ImmutableList.of(); this.originFunction = null; } + public CreateFunctionStmt(SetType type, boolean ifNotExists, FunctionName functionName, + FunctionArgsDef argsDef, + TypeDef returnType, TypeDef intermediateType, Map properties) { + this(type, ifNotExists, false, functionName, argsDef, returnType, intermediateType, properties); + this.isTableFunction = true; + } + public CreateFunctionStmt(SetType type, boolean ifNotExists, FunctionName functionName, FunctionArgsDef argsDef, List parameters, Expr originFunction) { this.type = type; @@ -158,6 +167,7 @@ public CreateFunctionStmt(SetType type, boolean ifNotExists, FunctionName functi } this.originFunction = originFunction; this.isAggregate = false; + this.isTableFunction = false; this.returnType = new TypeDef(Type.VARCHAR); this.properties = ImmutableSortedMap.of(); } @@ -197,6 +207,8 @@ public void analyze(Analyzer analyzer) throws UserException { analyzeUda(); } else if (isAlias) { analyzeAliasFunction(); + } else if (isTableFunction) { + analyzeTableFunction(); } else { analyzeUdf(); } @@ -208,6 +220,8 @@ public void analyze(Analyzer analyzer) throws UserException { analyzeUda(); } else if (isAlias) { analyzeAliasFunction(); + } else if (isTableFunction) { + analyzeTableFunction(); } else { analyzeUdf(); } @@ -301,6 +315,27 @@ private void computeObjectChecksum() throws IOException, NoSuchAlgorithmExceptio } } + private void analyzeTableFunction() throws AnalysisException { + String symbol = properties.get(SYMBOL_KEY); + if (Strings.isNullOrEmpty(symbol)) { + throw new AnalysisException("No 'symbol' in properties"); + } + if (!returnType.getType().isArrayType()) { + throw new AnalysisException("JAVA_UDF OF UDTF return type must be array type"); + } + analyzeJavaUdf(symbol); + URI location = URI.create(userFile); + function = ScalarFunction.createUdf(binaryType, + functionName, argsDef.getArgTypes(), + ((ArrayType) (returnType.getType())).getItemType(), argsDef.isVariadic(), + location, symbol, null, null); + function.setChecksum(checksum); + function.setNullableMode(returnNullMode); + function.setUDTFunction(true); + // Todo: maybe in create tables function, need register two function, one is + // normal and one is outer as those have different result when result is NULL. + } + private void analyzeUda() throws AnalysisException { AggregateFunction.AggregateFunctionBuilder builder = AggregateFunction.AggregateFunctionBuilder.createUdfBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 5253ff7fde15e7..14b56ac8b1f5b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1667,7 +1667,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { fn = getTableFunction(fnName.getFunction(), matchFuncChildTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); if (fn == null) { - throw new AnalysisException(getFunctionNotFoundError(argTypes)); + throw new AnalysisException(getFunctionNotFoundError(argTypes) + " in table function"); } // set param child types fn.setReturnType(((ArrayType) childTypes[0]).getItemType()); @@ -1675,6 +1675,16 @@ && collectChildReturnTypes()[0].isDecimalV3()) { fn = getTableFunction(fnName.getFunction(), childTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); } + // find user defined functions + if (fn == null) { + fn = findUdf(fnName, analyzer); + if (fn != null) { + FunctionUtil.checkEnableJavaUdf(); + if (!fn.isUDTFunction()) { + throw new AnalysisException(getFunctionNotFoundError(argTypes) + " in table function"); + } + } + } if (fn == null) { throw new AnalysisException(getFunctionNotFoundError(argTypes)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 0aa0da8ef36e0b..68a22aabb7f405 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -5323,6 +5323,15 @@ public void createFunction(CreateFunctionStmt stmt) throws UserException { } else { Database db = getInternalCatalog().getDbOrDdlException(stmt.getFunctionName().getDb()); db.addFunction(stmt.getFunction(), stmt.isIfNotExists()); + if (stmt.getFunction().isUDTFunction()) { + // all of the table function in doris will have two function + // one is the noraml, and another is outer, the different of them is deal with + // empty: whether need to insert NULL result value + Function outerFunction = stmt.getFunction().clone(); + FunctionName name = outerFunction.getFunctionName(); + name.setFn(name.getFunction() + "_outer"); + db.addFunction(outerFunction, stmt.isIfNotExists()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 7dbf3a0ec0a1a9..4d9c97e8dd9b4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -138,6 +138,8 @@ public enum NullableMode { // If true, this function is global function protected boolean isGlobal = false; + // If true, this function is table function, mainly used by java-udtf + protected boolean isUDTFunction = false; // Only used for serialization protected Function() { @@ -196,6 +198,8 @@ public Function(Function other) { System.arraycopy(other.argTypes, 0, this.argTypes, 0, other.argTypes.length); } this.checksum = other.checksum; + this.isGlobal = other.isGlobal; + this.isUDTFunction = other.isUDTFunction; } public void setNestedFunction(Function nestedFunction) { @@ -563,6 +567,7 @@ public TFunction toThrift(Type realReturnType, Type[] realArgTypes, Boolean[] re fn.setChecksum(checksum); } fn.setVectorized(vectorized); + fn.setIsUdtfFunction(isUDTFunction); return fn; } @@ -671,6 +676,7 @@ protected void writeFields(DataOutput output) throws IOException { IOUtils.writeOptionString(output, libUrl); IOUtils.writeOptionString(output, checksum); output.writeUTF(nullableMode.toString()); + output.writeBoolean(isUDTFunction); } @Override @@ -708,6 +714,9 @@ public void readFields(DataInput input) throws IOException { if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_126) { nullableMode = NullableMode.valueOf(input.readUTF()); } + if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_131) { + isUDTFunction = input.readBoolean(); + } } public static Function read(DataInput input) throws IOException { @@ -744,7 +753,11 @@ public List getInfo(boolean isVerbose) { // function type // intermediate type if (this instanceof ScalarFunction) { - row.add("Scalar"); + if (isUDTFunction()) { + row.add("TABLES"); + } else { + row.add("Scalar"); + } row.add("NULL"); } else if (this instanceof AliasFunction) { row.add("Alias"); @@ -775,6 +788,14 @@ public NullableMode getNullableMode() { return nullableMode; } + public void setUDTFunction(boolean isUDTFunction) { + this.isUDTFunction = isUDTFunction; + } + + public boolean isUDTFunction() { + return this.isUDTFunction; + } + // Try to serialize this function and write to nowhere. // Just for checking if we forget to implement write() method for some Exprs. // To avoid FE exist when writing edit log. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index e6c7e073579bdc..4c5dd85d4be07b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; +import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdtf; import org.apache.doris.nereids.types.DataType; import com.google.common.base.Strings; @@ -238,7 +239,11 @@ public static boolean translateToNereids(String dbName, Function function) { if (function instanceof AliasFunction) { AliasUdf.translateToNereidsFunction(dbName, ((AliasFunction) function)); } else if (function instanceof ScalarFunction) { - JavaUdf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); + if (function.isUDTFunction()) { + JavaUdtf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); + } else { + JavaUdf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); + } } else if (function instanceof AggregateFunction) { JavaUdaf.translateToNereidsFunction(dbName, ((AggregateFunction) function)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java index 31d97e9b536270..7d8368646f2b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java @@ -168,6 +168,21 @@ public static ScalarFunction createUdf( return fn; } + public ScalarFunction(ScalarFunction other) { + super(other); + if (other == null) { + return; + } + symbolName = other.symbolName; + prepareFnSymbol = other.prepareFnSymbol; + closeFnSymbol = other.closeFnSymbol; + } + + @Override + public Function clone() { + return new ScalarFunction(this); + } + public void setSymbolName(String s) { symbolName = s; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 6c7a1bd82c1078..70f1de8c55554b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -96,6 +96,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; +import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdtf; import org.apache.doris.nereids.trees.expressions.functions.window.WindowFunction; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -650,6 +651,14 @@ public Expr visitJavaUdf(JavaUdf udf, PlanTranslatorContext context) { return new FunctionCallExpr(udf.getCatalogFunction(), exprs); } + @Override + public Expr visitJavaUdtf(JavaUdtf udf, PlanTranslatorContext context) { + FunctionParams exprs = new FunctionParams(udf.children().stream() + .map(expression -> expression.accept(this, context)) + .collect(Collectors.toList())); + return new FunctionCallExpr(udf.getCatalogFunction(), exprs); + } + @Override public Expr visitJavaUdaf(JavaUdaf udaf, PlanTranslatorContext context) { FunctionParams exprs = new FunctionParams(udaf.isDistinct(), udaf.children().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java new file mode 100644 index 00000000000000..48bf65edc574ae --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.udf; + +import org.apache.doris.analysis.FunctionName; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.util.URI; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.thrift.TFunctionBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Java UDTF for Nereids + */ +public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCastableSignature, Udf { + private final String dbName; + private final long functionId; + private final TFunctionBinaryType binaryType; + private final FunctionSignature signature; + private final NullableMode nullableMode; + private final String objectFile; + private final String symbol; + private final String prepareFn; + private final String closeFn; + private final String checkSum; + + /** + * Constructor of UDTF + */ + public JavaUdtf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + FunctionSignature signature, + NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, + String checkSum, Expression... args) { + super(name, args); + this.dbName = dbName; + this.functionId = functionId; + this.binaryType = binaryType; + this.signature = signature; + this.nullableMode = nullableMode; + this.objectFile = objectFile; + this.symbol = symbol; + this.prepareFn = prepareFn; + this.closeFn = closeFn; + this.checkSum = checkSum; + } + + /** + * withChildren. + */ + @Override + public JavaUdtf withChildren(List children) { + Preconditions.checkArgument(children.size() == this.children.size()); + return new JavaUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, + objectFile, symbol, prepareFn, closeFn, checkSum, children.toArray(new Expression[0])); + } + + @Override + public List getSignatures() { + return ImmutableList.of(signature); + } + + @Override + public boolean hasVarArguments() { + return signature.hasVarArgs; + } + + @Override + public int arity() { + return signature.argumentsTypes.size(); + } + + @Override + public Function getCatalogFunction() { + try { + org.apache.doris.catalog.ScalarFunction expr = org.apache.doris.catalog.ScalarFunction.createUdf( + binaryType, + new FunctionName(dbName, getName()), + signature.argumentsTypes.stream().map(DataType::toCatalogDataType).toArray(Type[]::new), + signature.returnType.toCatalogDataType(), + signature.hasVarArgs, + URI.create(objectFile), + symbol, + prepareFn, + closeFn + ); + expr.setNullableMode(nullableMode); + expr.setChecksum(checkSum); + expr.setId(functionId); + expr.setUDTFunction(true); + return expr; + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } + } + + /** + * translate catalog java udf to nereids java udf + */ + public static void translateToNereidsFunction(String dbName, org.apache.doris.catalog.ScalarFunction scalar) { + String fnName = scalar.functionName(); + DataType retType = DataType.fromCatalogType(scalar.getReturnType()); + List argTypes = Arrays.stream(scalar.getArgs()) + .map(DataType::fromCatalogType) + .collect(Collectors.toList()); + + FunctionSignature.FuncSigBuilder sigBuilder = FunctionSignature.ret(retType); + FunctionSignature sig = scalar.hasVarArgs() + ? sigBuilder.varArgs(argTypes.toArray(new DataType[0])) + : sigBuilder.args(argTypes.toArray(new DataType[0])); + + VirtualSlotReference[] virtualSlots = argTypes.stream() + .map(type -> new VirtualSlotReference(type.toString(), type, Optional.empty(), + (shape) -> ImmutableList.of())) + .toArray(VirtualSlotReference[]::new); + + JavaUdtf udf = new JavaUdtf(fnName, scalar.getId(), dbName, scalar.getBinaryType(), sig, + scalar.getNullableMode(), + scalar.getLocation().getLocation(), + scalar.getSymbolName(), + scalar.getPrepareFnSymbol(), + scalar.getCloseFnSymbol(), + scalar.getChecksum(), + virtualSlots); + + JavaUdtfBuilder builder = new JavaUdtfBuilder(udf); + Env.getCurrentEnv().getFunctionRegistry().addUdf(dbName, fnName, builder); + } + + @Override + public NullableMode getNullableMode() { + return nullableMode; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitJavaUdtf(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtfBuilder.java new file mode 100644 index 00000000000000..85114f08e3f189 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtfBuilder.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.udf; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.ReflectionUtils; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import com.google.common.base.Suppliers; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * function builder for java udtf + */ +public class JavaUdtfBuilder extends UdfBuilder { + private final JavaUdtf udf; + private final int arity; + private final boolean isVarArgs; + + public JavaUdtfBuilder(JavaUdtf udf) { + this.udf = udf; + this.isVarArgs = udf.hasVarArguments(); + this.arity = udf.arity(); + } + + @Override + public List getArgTypes() { + return Suppliers.memoize(() -> udf.getSignatures().get(0).argumentsTypes.stream() + .map(DataType.class::cast) + .collect(Collectors.toList())).get(); + } + + @Override + public Class functionClass() { + return JavaUdtf.class; + } + + @Override + public boolean canApply(List arguments) { + if ((isVarArgs && arity > arguments.size() + 1) || (!isVarArgs && arguments.size() != arity)) { + return false; + } + for (Object argument : arguments) { + if (!(argument instanceof Expression)) { + Optional primitiveType = ReflectionUtils.getPrimitiveType(argument.getClass()); + if (!primitiveType.isPresent() || !Expression.class.isAssignableFrom(primitiveType.get())) { + return false; + } + } + } + return true; + } + + @Override + public Pair build(String name, List arguments) { + List exprs = arguments.stream().map(Expression.class::cast).collect(Collectors.toList()); + List argTypes = udf.getSignatures().get(0).argumentsTypes; + + List processedExprs = Lists.newArrayList(); + for (int i = 0; i < exprs.size(); ++i) { + processedExprs.add(TypeCoercionUtils.castIfNotSameType(exprs.get(i), argTypes.get(i))); + } + return Pair.ofSame(udf.withChildren(processedExprs)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index 4e4c8ab2bd492d..61042283a5180e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; +import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdtf; /** * visitor function for all table generating function. @@ -115,4 +116,7 @@ default R visitExplodeJsonArrayJsonOuter(ExplodeJsonArrayJsonOuter explodeJsonAr return visitTableGeneratingFunction(explodeJsonArrayJsonOuter, context); } + default R visitJavaUdtf(JavaUdtf udtf, C context) { + return visitTableGeneratingFunction(udtf, context); + } } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index d8953e7dd7a9b9..c949166ae5c47f 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -379,6 +379,7 @@ struct TFunction { 11: optional i64 id 12: optional string checksum 13: optional bool vectorized = false + 14: optional bool is_udtf_function = false } enum TJdbcOperation { diff --git a/regression-test/data/javaudf_p0/test_javaudtf_all_types.out b/regression-test/data/javaudf_p0/test_javaudtf_all_types.out new file mode 100644 index 00000000000000..96077f7537e3b9 --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_all_types.out @@ -0,0 +1,1390 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N +1 true 1 2 3 4 3.3300 7.77 3.1415 2023-10-18 2023-10-11T10:11:11.234 row1 [null, "nested1"] {"k1":null, "k2":"value1"} +2 false 2 4 6 8 1.6650 3.885 1.57075 2023-10-19 2023-10-12T10:12:11.234 row2 [null, "nested2"] {"k2":null, "k3":"value2"} +3 true 3 6 9 12 1.1100 2.59 1.04717 2023-10-20 2023-10-13T10:13:11.234 row3 [null, "nested3"] {"k3":null, "k4":"value3"} +4 false 4 8 12 16 0.8325 1.943 0.78538 2023-10-21 2023-10-14T10:14:11.234 row4 [null, "nested4"] {"k4":null, "k5":"value4"} +5 true 5 10 15 20 0.6660 1.554 0.6283 2023-10-22 2023-10-15T10:15:11.234 row5 [null, "nested5"] {"k5":null, "k6":"value5"} +6 false 6 12 18 24 0.5550 1.295 0.52358 2023-10-23 2023-10-16T10:16:11.234 row6 [null, "nested6"] {"k6":null, "k7":"value6"} +7 true 7 14 21 28 0.4757 1.11 0.44879 2023-10-24 2023-10-17T10:17:11.234 row7 [null, "nested7"] {"k7":null, "k8":"value7"} +8 false 8 16 24 32 0.4163 0.971 0.39269 2023-10-25 2023-10-18T10:18:11.234 row8 [null, "nested8"] {"k8":null, "k9":"value8"} +9 true 9 18 27 36 0.3700 0.863 0.34906 2023-10-26 2023-10-19T10:19:11.234 row9 [null, "nested9"] {"k9":null, "k10":"value9"} +10 false \N 20 30 40 \N 0.777 0.31415 \N 2023-10-20T10:10:11.234 \N [null, "nested10"] {"k10":null, "k11":"value10"} + +-- !select_boolean_col -- +1 true true +2 false false +2 false true +3 true false +3 true true +3 true true +4 false false +4 false false +4 false true +4 false true +5 true false +5 true false +5 true true +5 true true +5 true true +6 false false +6 false false +6 false false +6 false true +6 false true +6 false true +7 true false +7 true false +7 true false +7 true true +7 true true +7 true true +7 true true +8 false false +8 false false +8 false false +8 false false +8 false true +8 false true +8 false true +8 false true +9 true false +9 true false +9 true false +9 true false +9 true true +9 true true +9 true true +9 true true +9 true true +10 false false +10 false false +10 false false +10 false false +10 false false +10 false true +10 false true +10 false true +10 false true +10 false true + +-- !select_tinyint_col -- +1 1 1 +2 2 2 +2 2 3 +3 3 3 +3 3 4 +3 3 5 +4 4 4 +4 4 5 +4 4 6 +4 4 7 +5 5 5 +5 5 6 +5 5 7 +5 5 8 +5 5 9 +6 6 6 +6 6 7 +6 6 8 +6 6 9 +6 6 10 +6 6 11 +7 7 7 +7 7 8 +7 7 9 +7 7 10 +7 7 11 +7 7 12 +7 7 13 +8 8 8 +8 8 9 +8 8 10 +8 8 11 +8 8 12 +8 8 13 +8 8 14 +8 8 15 +9 9 9 +9 9 10 +9 9 11 +9 9 12 +9 9 13 +9 9 14 +9 9 15 +9 9 16 +9 9 17 + +-- !select_smallint_col -- +1 2 2 +2 4 4 +2 4 6 +3 6 6 +3 6 8 +3 6 10 +4 8 8 +4 8 10 +4 8 12 +4 8 14 +5 10 10 +5 10 12 +5 10 14 +5 10 16 +5 10 18 +6 12 12 +6 12 14 +6 12 16 +6 12 18 +6 12 20 +6 12 22 +7 14 14 +7 14 16 +7 14 18 +7 14 20 +7 14 22 +7 14 24 +7 14 26 +8 16 16 +8 16 18 +8 16 20 +8 16 22 +8 16 24 +8 16 26 +8 16 28 +8 16 30 +9 18 18 +9 18 20 +9 18 22 +9 18 24 +9 18 26 +9 18 28 +9 18 30 +9 18 32 +9 18 34 +10 20 20 +10 20 22 +10 20 24 +10 20 26 +10 20 28 +10 20 30 +10 20 32 +10 20 34 +10 20 36 +10 20 38 + +-- !select_int_col -- +1 1 1 +2 2 2 +2 2 5 +3 3 3 +3 3 6 +3 3 9 +4 4 4 +4 4 7 +4 4 10 +4 4 13 +5 5 5 +5 5 8 +5 5 11 +5 5 14 +5 5 17 +6 6 6 +6 6 9 +6 6 12 +6 6 15 +6 6 18 +6 6 21 +7 7 7 +7 7 10 +7 7 13 +7 7 16 +7 7 19 +7 7 22 +7 7 25 +8 8 8 +8 8 11 +8 8 14 +8 8 17 +8 8 20 +8 8 23 +8 8 26 +8 8 29 +9 9 9 +9 9 12 +9 9 15 +9 9 18 +9 9 21 +9 9 24 +9 9 27 +9 9 30 +9 9 33 +10 10 10 +10 10 13 +10 10 16 +10 10 19 +10 10 22 +10 10 25 +10 10 28 +10 10 31 +10 10 34 +10 10 37 + +-- !select_bigint_col -- +1 3 3 +2 6 6 +2 6 10 +3 9 9 +3 9 13 +3 9 17 +4 12 12 +4 12 16 +4 12 20 +4 12 24 +5 15 15 +5 15 19 +5 15 23 +5 15 27 +5 15 31 +6 18 18 +6 18 22 +6 18 26 +6 18 30 +6 18 34 +6 18 38 +7 21 21 +7 21 25 +7 21 29 +7 21 33 +7 21 37 +7 21 41 +7 21 45 +8 24 24 +8 24 28 +8 24 32 +8 24 36 +8 24 40 +8 24 44 +8 24 48 +8 24 52 +9 27 27 +9 27 31 +9 27 35 +9 27 39 +9 27 43 +9 27 47 +9 27 51 +9 27 55 +9 27 59 +10 30 30 +10 30 34 +10 30 38 +10 30 42 +10 30 46 +10 30 50 +10 30 54 +10 30 58 +10 30 62 +10 30 66 + +-- !select_largeint_col -- +1 4 4 +2 8 8 +2 8 13 +3 12 12 +3 12 17 +3 12 22 +4 16 16 +4 16 21 +4 16 26 +4 16 31 +5 20 20 +5 20 25 +5 20 30 +5 20 35 +5 20 40 +6 24 24 +6 24 29 +6 24 34 +6 24 39 +6 24 44 +6 24 49 +7 28 28 +7 28 33 +7 28 38 +7 28 43 +7 28 48 +7 28 53 +7 28 58 +8 32 32 +8 32 37 +8 32 42 +8 32 47 +8 32 52 +8 32 57 +8 32 62 +8 32 67 +9 36 36 +9 36 41 +9 36 46 +9 36 51 +9 36 56 +9 36 61 +9 36 66 +9 36 71 +9 36 76 +10 40 40 +10 40 45 +10 40 50 +10 40 55 +10 40 60 +10 40 65 +10 40 70 +10 40 75 +10 40 80 +10 40 85 + +-- !select_decimal_col -- +1 3.3300 3.3300 +2 1.6650 1.6650 +2 1.6650 1.6660 +3 1.1100 1.1100 +3 1.1100 1.1110 +3 1.1100 1.1120 +4 0.8325 0.8325 +4 0.8325 0.8335 +4 0.8325 0.8345 +4 0.8325 0.8355 +5 0.6660 0.6660 +5 0.6660 0.6670 +5 0.6660 0.6680 +5 0.6660 0.6690 +5 0.6660 0.6700 +6 0.5550 0.5550 +6 0.5550 0.5560 +6 0.5550 0.5570 +6 0.5550 0.5580 +6 0.5550 0.5590 +6 0.5550 0.5600 +7 0.4757 0.4757 +7 0.4757 0.4767 +7 0.4757 0.4777 +7 0.4757 0.4787 +7 0.4757 0.4797 +7 0.4757 0.4807 +7 0.4757 0.4817 +8 0.4163 0.4163 +8 0.4163 0.4173 +8 0.4163 0.4183 +8 0.4163 0.4193 +8 0.4163 0.4203 +8 0.4163 0.4213 +8 0.4163 0.4223 +8 0.4163 0.4233 +9 0.3700 0.3700 +9 0.3700 0.3710 +9 0.3700 0.3720 +9 0.3700 0.3730 +9 0.3700 0.3740 +9 0.3700 0.3750 +9 0.3700 0.3760 +9 0.3700 0.3770 +9 0.3700 0.3780 + +-- !select_float_col -- +1 7.77 7.77 +2 3.885 3.885 +2 3.885 3.985 +3 2.59 2.59 +3 2.59 2.6899998 +3 2.59 2.79 +4 1.943 1.943 +4 1.943 2.043 +4 1.943 2.143 +4 1.943 2.243 +5 1.554 1.554 +5 1.554 1.654 +5 1.554 1.7540001 +5 1.554 1.854 +5 1.554 1.954 +6 1.295 1.295 +6 1.295 1.395 +6 1.295 1.495 +6 1.295 1.5949999 +6 1.295 1.6949999 +6 1.295 1.795 +7 1.11 1.11 +7 1.11 1.21 +7 1.11 1.3100001 +7 1.11 1.41 +7 1.11 1.51 +7 1.11 1.61 +7 1.11 1.71 +8 0.971 0.971 +8 0.971 1.071 +8 0.971 1.171 +8 0.971 1.271 +8 0.971 1.371 +8 0.971 1.471 +8 0.971 1.571 +8 0.971 1.671 +9 0.863 0.863 +9 0.863 0.963 +9 0.863 1.063 +9 0.863 1.163 +9 0.863 1.263 +9 0.863 1.3629999 +9 0.863 1.4629999 +9 0.863 1.563 +9 0.863 1.663 +10 0.777 0.777 +10 0.777 0.87700003 +10 0.777 0.977 +10 0.777 1.077 +10 0.777 1.177 +10 0.777 1.277 +10 0.777 1.377 +10 0.777 1.477 +10 0.777 1.577 +10 0.777 1.677 + +-- !select_double_col -- +1 3.1415 3.1415 +2 1.57075 1.57075 +2 1.57075 1.58075 +3 1.04717 1.04717 +3 1.04717 1.05717 +3 1.04717 1.06717 +4 0.78538 0.78538 +4 0.78538 0.79538 +4 0.78538 0.80538 +4 0.78538 0.81538 +5 0.6283 0.6283 +5 0.6283 0.6383 +5 0.6283 0.6483 +5 0.6283 0.6583 +5 0.6283 0.6683 +6 0.52358 0.52358 +6 0.52358 0.53358 +6 0.52358 0.5435800000000001 +6 0.52358 0.5535800000000001 +6 0.52358 0.5635800000000001 +6 0.52358 0.5735800000000001 +7 0.44879 0.44879 +7 0.44879 0.45879000000000003 +7 0.44879 0.46879000000000004 +7 0.44879 0.47879000000000005 +7 0.44879 0.48879 +7 0.44879 0.49879 +7 0.44879 0.5087900000000001 +8 0.39269 0.39269 +8 0.39269 0.40269 +8 0.39269 0.41269 +8 0.39269 0.42269 +8 0.39269 0.43268999999999996 +8 0.39269 0.44269 +8 0.39269 0.45269 +8 0.39269 0.46269 +9 0.34906 0.34906 +9 0.34906 0.35906 +9 0.34906 0.36906 +9 0.34906 0.37905999999999995 +9 0.34906 0.38905999999999996 +9 0.34906 0.39905999999999997 +9 0.34906 0.40906 +9 0.34906 0.41906 +9 0.34906 0.42906 +10 0.31415 0.31415 +10 0.31415 0.32415 +10 0.31415 0.33415 +10 0.31415 0.34414999999999996 +10 0.31415 0.35414999999999996 +10 0.31415 0.36415 +10 0.31415 0.37415 +10 0.31415 0.38415 +10 0.31415 0.39415 +10 0.31415 0.40415 + +-- !select_date_col -- +1 2023-10-18 2023-10-18 +2 2023-10-19 2023-10-19 +2 2023-10-19 2023-11-19 +3 2023-10-20 2023-10-20 +3 2023-10-20 2023-11-20 +3 2023-10-20 2023-12-20 +4 2023-10-21 2023-10-21 +4 2023-10-21 2023-11-21 +4 2023-10-21 2023-12-21 +4 2023-10-21 2024-01-21 +5 2023-10-22 2023-10-22 +5 2023-10-22 2023-11-22 +5 2023-10-22 2023-12-22 +5 2023-10-22 2024-01-22 +5 2023-10-22 2024-02-22 +6 2023-10-23 2023-10-23 +6 2023-10-23 2023-11-23 +6 2023-10-23 2023-12-23 +6 2023-10-23 2024-01-23 +6 2023-10-23 2024-02-23 +6 2023-10-23 2024-03-23 +7 2023-10-24 2023-10-24 +7 2023-10-24 2023-11-24 +7 2023-10-24 2023-12-24 +7 2023-10-24 2024-01-24 +7 2023-10-24 2024-02-24 +7 2023-10-24 2024-03-24 +7 2023-10-24 2024-04-24 +8 2023-10-25 2023-10-25 +8 2023-10-25 2023-11-25 +8 2023-10-25 2023-12-25 +8 2023-10-25 2024-01-25 +8 2023-10-25 2024-02-25 +8 2023-10-25 2024-03-25 +8 2023-10-25 2024-04-25 +8 2023-10-25 2024-05-25 +9 2023-10-26 2023-10-26 +9 2023-10-26 2023-11-26 +9 2023-10-26 2023-12-26 +9 2023-10-26 2024-01-26 +9 2023-10-26 2024-02-26 +9 2023-10-26 2024-03-26 +9 2023-10-26 2024-04-26 +9 2023-10-26 2024-05-26 +9 2023-10-26 2024-06-26 + +-- !select_datetime_col -- +1 2023-10-11T10:11:11.234 2023-10-11T10:11:11.234 +2 2023-10-12T10:12:11.234 2023-10-12T10:12:11.234 +2 2023-10-12T10:12:11.234 2023-10-13T10:12:11.234 +3 2023-10-13T10:13:11.234 2023-10-13T10:13:11.234 +3 2023-10-13T10:13:11.234 2023-10-14T10:13:11.234 +3 2023-10-13T10:13:11.234 2023-10-15T10:13:11.234 +4 2023-10-14T10:14:11.234 2023-10-14T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-15T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-16T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-17T10:14:11.234 +5 2023-10-15T10:15:11.234 2023-10-15T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-16T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-17T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-18T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-19T10:15:11.234 +6 2023-10-16T10:16:11.234 2023-10-16T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-17T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-18T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-19T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-20T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-21T10:16:11.234 +7 2023-10-17T10:17:11.234 2023-10-17T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-18T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-19T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-20T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-21T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-22T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-23T10:17:11.234 +8 2023-10-18T10:18:11.234 2023-10-18T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-19T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-20T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-21T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-22T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-23T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-24T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-25T10:18:11.234 +9 2023-10-19T10:19:11.234 2023-10-19T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-20T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-21T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-22T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-23T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-24T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-25T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-26T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-27T10:19:11.234 +10 2023-10-20T10:10:11.234 2023-10-20T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-21T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-22T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-23T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-24T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-25T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-26T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-27T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-28T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-29T10:10:11.234 + +-- !select_string_col -- +1 row1 1 +1 row1 o +1 row1 r +1 row1 w +2 row2 2 +2 row2 o +2 row2 r +2 row2 w +3 row3 3 +3 row3 o +3 row3 r +3 row3 w +4 row4 4 +4 row4 o +4 row4 r +4 row4 w +5 row5 5 +5 row5 o +5 row5 r +5 row5 w +6 row6 6 +6 row6 o +6 row6 r +6 row6 w +7 row7 7 +7 row7 o +7 row7 r +7 row7 w +8 row8 8 +8 row8 o +8 row8 r +8 row8 w +9 row9 9 +9 row9 o +9 row9 r +9 row9 w + +-- !select_array_col -- +1 [null, "nested1"] \N +1 [null, "nested1"] nested1 +2 [null, "nested2"] \N +2 [null, "nested2"] nested2 +3 [null, "nested3"] \N +3 [null, "nested3"] nested3 +4 [null, "nested4"] \N +4 [null, "nested4"] nested4 +5 [null, "nested5"] \N +5 [null, "nested5"] nested5 +6 [null, "nested6"] \N +6 [null, "nested6"] nested6 +7 [null, "nested7"] \N +7 [null, "nested7"] nested7 +8 [null, "nested8"] \N +8 [null, "nested8"] nested8 +9 [null, "nested9"] \N +9 [null, "nested9"] nested9 +10 [null, "nested10"] \N +10 [null, "nested10"] nested10 + +-- !select_map_col -- +1 {"k1":null, "k2":"value1"} k1 +1 {"k1":null, "k2":"value1"} k2 +2 {"k2":null, "k3":"value2"} \N +2 {"k2":null, "k3":"value2"} value2 +3 {"k3":null, "k4":"value3"} k3 +3 {"k3":null, "k4":"value3"} k4 +4 {"k4":null, "k5":"value4"} \N +4 {"k4":null, "k5":"value4"} value4 +5 {"k5":null, "k6":"value5"} k5 +5 {"k5":null, "k6":"value5"} k6 +6 {"k6":null, "k7":"value6"} \N +6 {"k6":null, "k7":"value6"} value6 +7 {"k7":null, "k8":"value7"} k7 +7 {"k7":null, "k8":"value7"} k8 +8 {"k8":null, "k9":"value8"} \N +8 {"k8":null, "k9":"value8"} value8 +9 {"k9":null, "k10":"value9"} k10 +9 {"k9":null, "k10":"value9"} k9 +10 {"k10":null, "k11":"value10"} \N +10 {"k10":null, "k11":"value10"} value10 + +-- !select_boolean_col_outer -- +\N \N \N +1 true true +2 false false +2 false true +3 true false +3 true true +3 true true +4 false false +4 false false +4 false true +4 false true +5 true false +5 true false +5 true true +5 true true +5 true true +6 false false +6 false false +6 false false +6 false true +6 false true +6 false true +7 true false +7 true false +7 true false +7 true true +7 true true +7 true true +7 true true +8 false false +8 false false +8 false false +8 false false +8 false true +8 false true +8 false true +8 false true +9 true false +9 true false +9 true false +9 true false +9 true true +9 true true +9 true true +9 true true +9 true true +10 false false +10 false false +10 false false +10 false false +10 false false +10 false true +10 false true +10 false true +10 false true +10 false true + +-- !select_tinyint_col_outer -- +\N \N \N +1 1 1 +2 2 2 +2 2 3 +3 3 3 +3 3 4 +3 3 5 +4 4 4 +4 4 5 +4 4 6 +4 4 7 +5 5 5 +5 5 6 +5 5 7 +5 5 8 +5 5 9 +6 6 6 +6 6 7 +6 6 8 +6 6 9 +6 6 10 +6 6 11 +7 7 7 +7 7 8 +7 7 9 +7 7 10 +7 7 11 +7 7 12 +7 7 13 +8 8 8 +8 8 9 +8 8 10 +8 8 11 +8 8 12 +8 8 13 +8 8 14 +8 8 15 +9 9 9 +9 9 10 +9 9 11 +9 9 12 +9 9 13 +9 9 14 +9 9 15 +9 9 16 +9 9 17 +10 \N \N + +-- !select_smallint_col_outer -- +\N \N \N +1 2 2 +2 4 4 +2 4 6 +3 6 6 +3 6 8 +3 6 10 +4 8 8 +4 8 10 +4 8 12 +4 8 14 +5 10 10 +5 10 12 +5 10 14 +5 10 16 +5 10 18 +6 12 12 +6 12 14 +6 12 16 +6 12 18 +6 12 20 +6 12 22 +7 14 14 +7 14 16 +7 14 18 +7 14 20 +7 14 22 +7 14 24 +7 14 26 +8 16 16 +8 16 18 +8 16 20 +8 16 22 +8 16 24 +8 16 26 +8 16 28 +8 16 30 +9 18 18 +9 18 20 +9 18 22 +9 18 24 +9 18 26 +9 18 28 +9 18 30 +9 18 32 +9 18 34 +10 20 20 +10 20 22 +10 20 24 +10 20 26 +10 20 28 +10 20 30 +10 20 32 +10 20 34 +10 20 36 +10 20 38 + +-- !select_int_col_outer -- +\N \N \N +1 1 1 +2 2 2 +2 2 5 +3 3 3 +3 3 6 +3 3 9 +4 4 4 +4 4 7 +4 4 10 +4 4 13 +5 5 5 +5 5 8 +5 5 11 +5 5 14 +5 5 17 +6 6 6 +6 6 9 +6 6 12 +6 6 15 +6 6 18 +6 6 21 +7 7 7 +7 7 10 +7 7 13 +7 7 16 +7 7 19 +7 7 22 +7 7 25 +8 8 8 +8 8 11 +8 8 14 +8 8 17 +8 8 20 +8 8 23 +8 8 26 +8 8 29 +9 9 9 +9 9 12 +9 9 15 +9 9 18 +9 9 21 +9 9 24 +9 9 27 +9 9 30 +9 9 33 +10 10 10 +10 10 13 +10 10 16 +10 10 19 +10 10 22 +10 10 25 +10 10 28 +10 10 31 +10 10 34 +10 10 37 + +-- !select_bigint_col_outer -- +\N \N \N +1 3 3 +2 6 6 +2 6 10 +3 9 9 +3 9 13 +3 9 17 +4 12 12 +4 12 16 +4 12 20 +4 12 24 +5 15 15 +5 15 19 +5 15 23 +5 15 27 +5 15 31 +6 18 18 +6 18 22 +6 18 26 +6 18 30 +6 18 34 +6 18 38 +7 21 21 +7 21 25 +7 21 29 +7 21 33 +7 21 37 +7 21 41 +7 21 45 +8 24 24 +8 24 28 +8 24 32 +8 24 36 +8 24 40 +8 24 44 +8 24 48 +8 24 52 +9 27 27 +9 27 31 +9 27 35 +9 27 39 +9 27 43 +9 27 47 +9 27 51 +9 27 55 +9 27 59 +10 30 30 +10 30 34 +10 30 38 +10 30 42 +10 30 46 +10 30 50 +10 30 54 +10 30 58 +10 30 62 +10 30 66 + +-- !select_largeint_col_outer -- +\N \N \N +1 4 4 +2 8 8 +2 8 13 +3 12 12 +3 12 17 +3 12 22 +4 16 16 +4 16 21 +4 16 26 +4 16 31 +5 20 20 +5 20 25 +5 20 30 +5 20 35 +5 20 40 +6 24 24 +6 24 29 +6 24 34 +6 24 39 +6 24 44 +6 24 49 +7 28 28 +7 28 33 +7 28 38 +7 28 43 +7 28 48 +7 28 53 +7 28 58 +8 32 32 +8 32 37 +8 32 42 +8 32 47 +8 32 52 +8 32 57 +8 32 62 +8 32 67 +9 36 36 +9 36 41 +9 36 46 +9 36 51 +9 36 56 +9 36 61 +9 36 66 +9 36 71 +9 36 76 +10 40 40 +10 40 45 +10 40 50 +10 40 55 +10 40 60 +10 40 65 +10 40 70 +10 40 75 +10 40 80 +10 40 85 + +-- !select_decimal_col_outer -- +\N \N \N +1 3.3300 3.3300 +2 1.6650 1.6650 +2 1.6650 1.6660 +3 1.1100 1.1100 +3 1.1100 1.1110 +3 1.1100 1.1120 +4 0.8325 0.8325 +4 0.8325 0.8335 +4 0.8325 0.8345 +4 0.8325 0.8355 +5 0.6660 0.6660 +5 0.6660 0.6670 +5 0.6660 0.6680 +5 0.6660 0.6690 +5 0.6660 0.6700 +6 0.5550 0.5550 +6 0.5550 0.5560 +6 0.5550 0.5570 +6 0.5550 0.5580 +6 0.5550 0.5590 +6 0.5550 0.5600 +7 0.4757 0.4757 +7 0.4757 0.4767 +7 0.4757 0.4777 +7 0.4757 0.4787 +7 0.4757 0.4797 +7 0.4757 0.4807 +7 0.4757 0.4817 +8 0.4163 0.4163 +8 0.4163 0.4173 +8 0.4163 0.4183 +8 0.4163 0.4193 +8 0.4163 0.4203 +8 0.4163 0.4213 +8 0.4163 0.4223 +8 0.4163 0.4233 +9 0.3700 0.3700 +9 0.3700 0.3710 +9 0.3700 0.3720 +9 0.3700 0.3730 +9 0.3700 0.3740 +9 0.3700 0.3750 +9 0.3700 0.3760 +9 0.3700 0.3770 +9 0.3700 0.3780 +10 \N \N + +-- !select_float_col_outer -- +\N \N \N +1 7.77 7.77 +2 3.885 3.885 +2 3.885 3.985 +3 2.59 2.59 +3 2.59 2.6899998 +3 2.59 2.79 +4 1.943 1.943 +4 1.943 2.043 +4 1.943 2.143 +4 1.943 2.243 +5 1.554 1.554 +5 1.554 1.654 +5 1.554 1.7540001 +5 1.554 1.854 +5 1.554 1.954 +6 1.295 1.295 +6 1.295 1.395 +6 1.295 1.495 +6 1.295 1.5949999 +6 1.295 1.6949999 +6 1.295 1.795 +7 1.11 1.11 +7 1.11 1.21 +7 1.11 1.3100001 +7 1.11 1.41 +7 1.11 1.51 +7 1.11 1.61 +7 1.11 1.71 +8 0.971 0.971 +8 0.971 1.071 +8 0.971 1.171 +8 0.971 1.271 +8 0.971 1.371 +8 0.971 1.471 +8 0.971 1.571 +8 0.971 1.671 +9 0.863 0.863 +9 0.863 0.963 +9 0.863 1.063 +9 0.863 1.163 +9 0.863 1.263 +9 0.863 1.3629999 +9 0.863 1.4629999 +9 0.863 1.563 +9 0.863 1.663 +10 0.777 0.777 +10 0.777 0.87700003 +10 0.777 0.977 +10 0.777 1.077 +10 0.777 1.177 +10 0.777 1.277 +10 0.777 1.377 +10 0.777 1.477 +10 0.777 1.577 +10 0.777 1.677 + +-- !select_double_col_outer -- +\N \N \N +1 3.1415 3.1415 +2 1.57075 1.57075 +2 1.57075 1.58075 +3 1.04717 1.04717 +3 1.04717 1.05717 +3 1.04717 1.06717 +4 0.78538 0.78538 +4 0.78538 0.79538 +4 0.78538 0.80538 +4 0.78538 0.81538 +5 0.6283 0.6283 +5 0.6283 0.6383 +5 0.6283 0.6483 +5 0.6283 0.6583 +5 0.6283 0.6683 +6 0.52358 0.52358 +6 0.52358 0.53358 +6 0.52358 0.5435800000000001 +6 0.52358 0.5535800000000001 +6 0.52358 0.5635800000000001 +6 0.52358 0.5735800000000001 +7 0.44879 0.44879 +7 0.44879 0.45879000000000003 +7 0.44879 0.46879000000000004 +7 0.44879 0.47879000000000005 +7 0.44879 0.48879 +7 0.44879 0.49879 +7 0.44879 0.5087900000000001 +8 0.39269 0.39269 +8 0.39269 0.40269 +8 0.39269 0.41269 +8 0.39269 0.42269 +8 0.39269 0.43268999999999996 +8 0.39269 0.44269 +8 0.39269 0.45269 +8 0.39269 0.46269 +9 0.34906 0.34906 +9 0.34906 0.35906 +9 0.34906 0.36906 +9 0.34906 0.37905999999999995 +9 0.34906 0.38905999999999996 +9 0.34906 0.39905999999999997 +9 0.34906 0.40906 +9 0.34906 0.41906 +9 0.34906 0.42906 +10 0.31415 0.31415 +10 0.31415 0.32415 +10 0.31415 0.33415 +10 0.31415 0.34414999999999996 +10 0.31415 0.35414999999999996 +10 0.31415 0.36415 +10 0.31415 0.37415 +10 0.31415 0.38415 +10 0.31415 0.39415 +10 0.31415 0.40415 + +-- !select_date_col_outer -- +\N \N \N +1 2023-10-18 2023-10-18 +2 2023-10-19 2023-10-19 +2 2023-10-19 2023-11-19 +3 2023-10-20 2023-10-20 +3 2023-10-20 2023-11-20 +3 2023-10-20 2023-12-20 +4 2023-10-21 2023-10-21 +4 2023-10-21 2023-11-21 +4 2023-10-21 2023-12-21 +4 2023-10-21 2024-01-21 +5 2023-10-22 2023-10-22 +5 2023-10-22 2023-11-22 +5 2023-10-22 2023-12-22 +5 2023-10-22 2024-01-22 +5 2023-10-22 2024-02-22 +6 2023-10-23 2023-10-23 +6 2023-10-23 2023-11-23 +6 2023-10-23 2023-12-23 +6 2023-10-23 2024-01-23 +6 2023-10-23 2024-02-23 +6 2023-10-23 2024-03-23 +7 2023-10-24 2023-10-24 +7 2023-10-24 2023-11-24 +7 2023-10-24 2023-12-24 +7 2023-10-24 2024-01-24 +7 2023-10-24 2024-02-24 +7 2023-10-24 2024-03-24 +7 2023-10-24 2024-04-24 +8 2023-10-25 2023-10-25 +8 2023-10-25 2023-11-25 +8 2023-10-25 2023-12-25 +8 2023-10-25 2024-01-25 +8 2023-10-25 2024-02-25 +8 2023-10-25 2024-03-25 +8 2023-10-25 2024-04-25 +8 2023-10-25 2024-05-25 +9 2023-10-26 2023-10-26 +9 2023-10-26 2023-11-26 +9 2023-10-26 2023-12-26 +9 2023-10-26 2024-01-26 +9 2023-10-26 2024-02-26 +9 2023-10-26 2024-03-26 +9 2023-10-26 2024-04-26 +9 2023-10-26 2024-05-26 +9 2023-10-26 2024-06-26 +10 \N \N + +-- !select_datetime_col_outer -- +\N \N \N +1 2023-10-11T10:11:11.234 2023-10-11T10:11:11.234 +2 2023-10-12T10:12:11.234 2023-10-12T10:12:11.234 +2 2023-10-12T10:12:11.234 2023-10-13T10:12:11.234 +3 2023-10-13T10:13:11.234 2023-10-13T10:13:11.234 +3 2023-10-13T10:13:11.234 2023-10-14T10:13:11.234 +3 2023-10-13T10:13:11.234 2023-10-15T10:13:11.234 +4 2023-10-14T10:14:11.234 2023-10-14T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-15T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-16T10:14:11.234 +4 2023-10-14T10:14:11.234 2023-10-17T10:14:11.234 +5 2023-10-15T10:15:11.234 2023-10-15T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-16T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-17T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-18T10:15:11.234 +5 2023-10-15T10:15:11.234 2023-10-19T10:15:11.234 +6 2023-10-16T10:16:11.234 2023-10-16T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-17T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-18T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-19T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-20T10:16:11.234 +6 2023-10-16T10:16:11.234 2023-10-21T10:16:11.234 +7 2023-10-17T10:17:11.234 2023-10-17T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-18T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-19T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-20T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-21T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-22T10:17:11.234 +7 2023-10-17T10:17:11.234 2023-10-23T10:17:11.234 +8 2023-10-18T10:18:11.234 2023-10-18T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-19T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-20T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-21T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-22T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-23T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-24T10:18:11.234 +8 2023-10-18T10:18:11.234 2023-10-25T10:18:11.234 +9 2023-10-19T10:19:11.234 2023-10-19T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-20T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-21T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-22T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-23T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-24T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-25T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-26T10:19:11.234 +9 2023-10-19T10:19:11.234 2023-10-27T10:19:11.234 +10 2023-10-20T10:10:11.234 2023-10-20T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-21T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-22T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-23T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-24T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-25T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-26T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-27T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-28T10:10:11.234 +10 2023-10-20T10:10:11.234 2023-10-29T10:10:11.234 + +-- !select_string_col_outer -- +\N \N \N +1 row1 1 +1 row1 o +1 row1 r +1 row1 w +2 row2 2 +2 row2 o +2 row2 r +2 row2 w +3 row3 3 +3 row3 o +3 row3 r +3 row3 w +4 row4 4 +4 row4 o +4 row4 r +4 row4 w +5 row5 5 +5 row5 o +5 row5 r +5 row5 w +6 row6 6 +6 row6 o +6 row6 r +6 row6 w +7 row7 7 +7 row7 o +7 row7 r +7 row7 w +8 row8 8 +8 row8 o +8 row8 r +8 row8 w +9 row9 9 +9 row9 o +9 row9 r +9 row9 w +10 \N \N + +-- !select_array_col_outer -- +\N \N \N +1 [null, "nested1"] \N +1 [null, "nested1"] nested1 +2 [null, "nested2"] \N +2 [null, "nested2"] nested2 +3 [null, "nested3"] \N +3 [null, "nested3"] nested3 +4 [null, "nested4"] \N +4 [null, "nested4"] nested4 +5 [null, "nested5"] \N +5 [null, "nested5"] nested5 +6 [null, "nested6"] \N +6 [null, "nested6"] nested6 +7 [null, "nested7"] \N +7 [null, "nested7"] nested7 +8 [null, "nested8"] \N +8 [null, "nested8"] nested8 +9 [null, "nested9"] \N +9 [null, "nested9"] nested9 +10 [null, "nested10"] \N +10 [null, "nested10"] nested10 + +-- !select_map_col_outer -- +\N \N \N +1 {"k1":null, "k2":"value1"} k1 +1 {"k1":null, "k2":"value1"} k2 +2 {"k2":null, "k3":"value2"} \N +2 {"k2":null, "k3":"value2"} value2 +3 {"k3":null, "k4":"value3"} k3 +3 {"k3":null, "k4":"value3"} k4 +4 {"k4":null, "k5":"value4"} \N +4 {"k4":null, "k5":"value4"} value4 +5 {"k5":null, "k6":"value5"} k5 +5 {"k5":null, "k6":"value5"} k6 +6 {"k6":null, "k7":"value6"} \N +6 {"k6":null, "k7":"value6"} value6 +7 {"k7":null, "k8":"value7"} k7 +7 {"k7":null, "k8":"value7"} k8 +8 {"k8":null, "k9":"value8"} \N +8 {"k8":null, "k9":"value8"} value8 +9 {"k9":null, "k10":"value9"} k10 +9 {"k9":null, "k10":"value9"} k9 +10 {"k10":null, "k11":"value10"} \N +10 {"k10":null, "k11":"value10"} value10 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out b/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out new file mode 100644 index 00000000000000..c7cf111f2e682e --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 2 2022-01-01 2022-01-01T11:11:11 a1b +2 4 2022-01-01 2022-01-01T11:11:11 a2b +3 6 2022-01-01 2022-01-01T11:11:11 a3b +4 8 2022-01-01 2022-01-01T11:11:11 a4b +5 10 2022-01-01 2022-01-01T11:11:11 a5b +6 12 2022-01-01 2022-01-01T11:11:11 a6b +7 14 2022-01-01 2022-01-01T11:11:11 a7b +8 16 2022-01-01 2022-01-01T11:11:11 a8b +9 18 2022-01-01 2022-01-01T11:11:11 a9b +10 20 2022-06-06 2022-01-01T12:12:12 a10b + +-- !select1 -- +1 a1b 1 +2 a2b 2 +3 a3b 3 +4 a4b 4 +5 a5b 5 +6 a6b 6 +7 a7b 7 +8 a8b 8 +9 a9b 9 +10 a10b 10 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_decimal.out b/regression-test/data/javaudf_p0/test_javaudtf_decimal.out new file mode 100644 index 00000000000000..4a0fcd24181601 --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_decimal.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +111 11111.111110000 222222.333333300 +112 1234556.111110000 222222.333333300 +113 87654321.111110000 \N + +-- !select1 -- +111 11111.111110000 22222.222220000 +112 1234556.111110000 2469112.222220000 +113 87654321.111110000 175308642.222220000 + +-- !select2 -- +111 222222.333333300 444444.666666600 +112 222222.333333300 444444.666666600 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_double.out b/regression-test/data/javaudf_p0/test_javaudtf_double.out new file mode 100644 index 00000000000000..0ce1c152d0e62e --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_double.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +111 11111.111 222222.33 1.234567834455677E7 1111112.0 +112 1234556.1 222222.33 2.2222222233333334E8 4.444444444444556E12 +113 8.765432E7 \N 6.666666666666667E9 \N + +-- !select1 -- +111 1.234567834455677E7 1.234567834455677E8 +112 2.2222222233333334E8 2.2222222233333335E9 +113 6.666666666666667E9 6.666666666666667E10 + +-- !select2 -- +111 1111112.0 1.111112E7 +112 4.444444444444556E12 4.4444444444445555E13 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_int.out b/regression-test/data/javaudf_p0/test_javaudtf_int.out new file mode 100644 index 00000000000000..2fa8be4ac94002 --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_int.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 1 abc,defg poiuytre,abcdefg +2 2 abc,defg poiuytre,abcdefg +0 3 abc,defg poiuytre,abcdefg +1 4 abc,defg poiuytre,abcdefg +2 5 abc,defg poiuytre,abcdefg +0 6 abc,defg poiuytre,abcdefg +1 7 abc,defg poiuytre,abcdefg +2 8 abc,defg poiuytre,abcdefg +9 9 ab,cdefg poiuytreabcde,fg + +-- !select1 -- +1 abc,defg 1 +1 abc,defg 1 +1 abc,defg 1 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_string.out b/regression-test/data/javaudf_p0/test_javaudtf_string.out new file mode 100644 index 00000000000000..7fc0036c5bfbab --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_string.out @@ -0,0 +1,34 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 1 abc,defg poiuytre,abcdefg +2 2 abc,defg poiuytre,abcdefg +0 3 abc,defg poiuytre,abcdefg +1 4 abc,defg poiuytre,abcdefg +2 5 abc,defg poiuytre,abcdefg +0 6 abc,defg poiuytre,abcdefg +1 7 abc,defg poiuytre,abcdefg +2 8 abc,defg poiuytre,abcdefg +9 9 ab,cdefg poiuytreabcde,fg + +-- !select1 -- +0 abc,defg abc +0 abc,defg defg +0 abc,defg abc +0 abc,defg defg +1 abc,defg abc +1 abc,defg defg +1 abc,defg abc +1 abc,defg defg +1 abc,defg abc +1 abc,defg defg +2 abc,defg abc +2 abc,defg defg +2 abc,defg abc +2 abc,defg defg +2 abc,defg abc +2 abc,defg defg +9 ab,cdefg ab +9 ab,cdefg cdefg + +-- !select2 -- + diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFAllTypeTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFAllTypeTest.java new file mode 100644 index 00000000000000..b3060d1d438e4c --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFAllTypeTest.java @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashMap; + +public class UDTFAllTypeTest { + public static class UdtfBoolean { + public ArrayList evaluate(Boolean value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add((i % 2 == 0)); + } + return result; + } + } + + public static class UdtfByte { + public ArrayList evaluate(Byte value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add((byte) (value + i)); + } + return result; + } + } + + public static class UdtfShort { + public ArrayList evaluate(Short value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add((short) (value + i * 2)); + } + return result; + } + } + + public static class UdtfInt { + public ArrayList evaluate(Integer value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(value + i * 3); + } + return result; + } + } + + public static class UdtfLong { + public ArrayList evaluate(Long value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add((long) (value + i * 4)); + } + return result; + } + } + + public static class UdtfLargeInt { + public ArrayList evaluate(BigInteger value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(new BigInteger(String.valueOf(i * 5)).add(value)); + } + return result; + } + } + + public static class UdtfFloat { + public ArrayList evaluate(Float value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add((float) (value + i * 0.1)); + } + return result; + } + } + + public static class UdtfDouble { + public ArrayList evaluate(Double value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(value + i * 0.01); + } + return result; + } + } + + public static class UdtfDecimal { + public ArrayList evaluate(BigDecimal value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(new BigDecimal(String.valueOf(i * 0.001)).add(value)); + } + return result; + } + } + + public static class UdtfDate { + public ArrayList evaluate(LocalDate value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(value.plusMonths(i)); + } + return result; + } + } + + public static class UdtfDateTime { + public ArrayList evaluate(LocalDateTime value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + result.add(value.plusDays(i)); + } + return result; + } + } + + public static class UdtfString { + public ArrayList evaluate(String value, String separator) { + if (value == null || separator == null) { + return null; + } else { + return new ArrayList<>(Arrays.asList(value.split(separator))); + } + } + } + + public static class UdtfList { + public ArrayList evaluate(ArrayList value, Integer count) { + if (value == null || count == null) { + return null; + } + if (count % 2 == 1) { + Collections.reverse(value); + } + return value; + } + } + + public static class UdtfMap { + public ArrayList evaluate(HashMap value, Integer count) { + if (value == null || count == null) { + return null; + } + ArrayList result; + if (count % 2 == 1) { + result = new ArrayList<>(value.keySet()); + } else { + result = new ArrayList<>(value.values()); + } + return result; + } + } + +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java new file mode 100644 index 00000000000000..00fb948c826c2f --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFArrayIntTest { + public ArrayList evaluate(ArrayList val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + for (int i = 0; i < val.size(); i = i + 2) { + result.add(val.get(i)); + } + return val; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java new file mode 100644 index 00000000000000..9f8d3c07ddc359 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.math.BigDecimal; +import java.util.ArrayList; + +public class UDTFDecimalTest { + public ArrayList evaluate(BigDecimal val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + BigDecimal sum = val.add(val); + result.add(sum); + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java new file mode 100644 index 00000000000000..359df11ebbdae4 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFDoubleTest { + public ArrayList evaluate(Double val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + result.add(val * 10); + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java new file mode 100644 index 00000000000000..27b435727bef3c --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFIntTest { + public ArrayList evaluate(int count) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; i++) { + result.add(count); + } + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java new file mode 100644 index 00000000000000..dc6ac9be90c029 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFNullTest { + public ArrayList evaluate(String value, String separator) { + return null; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java new file mode 100644 index 00000000000000..cb2eb45c9c1f92 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; +import java.util.Arrays; + +public class UDTFStringTest { + public ArrayList evaluate(String value, String separator) { + if (value == null || separator == null) { + return null; + } else { + return new ArrayList<>(Arrays.asList(value.split(separator))); + } + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_all_types.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_all_types.groovy new file mode 100644 index 00000000000000..aaf9d4d5b429e3 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_all_types.groovy @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_all_types") { + def tableName = "test_javaudtf_all_types" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + int_col int, + boolean_col boolean, + tinyint_col tinyint, + smallint_col smallint, + bigint_col bigint, + largeint_col largeint, + decimal_col decimal(15, 4), + float_col float, + double_col double, + date_col date, + datetime_col datetime(6), + string_col string, + array_col array, + map_col map + ) + DISTRIBUTED BY HASH(int_col) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i++) { + sb.append(""" + (${i},${i%2},${i},${i}*2,${i}*3,${i}*4,${3.33/i},${(7.77/i).round(3)},${(3.1415/i).round(5)},"2023-10-${i+17}","2023-10-${i+10} 10:1${i}:11.234","row${i}",array(null, "nested${i}"),{"k${i}":null,"k${i+1}":"value${i}"}), + """) + } + sb.append(""" + (${i},${i%2},null,${i}*2,${i}*3,${i}*4,null,${(7.77/i).round(3)},${(3.1415/i).round(5)},null,"2023-10-${i+10} 10:${i}:11.234",null,array(null, "nested${i}"),{"k${i}":null,"k${i+1}":"value${i}"}), + """) + sb.append(""" + (null,null,null,null,null,null,null,null,null,null,null,null,null,null) + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + qt_select """select * from ${tableName} order by 1,2,3;""" + + + sql """DROP FUNCTION IF EXISTS udtf_boolean(boolean, int);""" + sql """CREATE TABLES FUNCTION udtf_boolean(boolean, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfBoolean", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_tinyint(tinyint, int);""" + sql """CREATE TABLES FUNCTION udtf_tinyint(tinyint, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfByte", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_short(smallint, int);""" + sql """CREATE TABLES FUNCTION udtf_short(smallint, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfShort", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_int(int, int);""" + sql """CREATE TABLES FUNCTION udtf_int(int, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfInt", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_long(bigint, int);""" + sql """CREATE TABLES FUNCTION udtf_long(bigint, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfLong", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_largeint(largeint, int);""" + sql """CREATE TABLES FUNCTION udtf_largeint(largeint, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfLargeInt", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_decimal(decimal(15, 4), int);""" + sql """CREATE TABLES FUNCTION udtf_decimal(decimal(15, 4), int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfDecimal", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_float(float, int);""" + sql """CREATE TABLES FUNCTION udtf_float(float, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfFloat", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_double(double, int);""" + sql """CREATE TABLES FUNCTION udtf_double(double, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfDouble", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_date(date, int);""" + sql """CREATE TABLES FUNCTION udtf_date(date, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfDate", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_datetime(datetime(6), int);""" + sql """CREATE TABLES FUNCTION udtf_datetime(datetime(6), int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfDateTime", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_string(string, string);""" + sql """CREATE TABLES FUNCTION udtf_string(string,string) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfString", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_list(array, int);""" + sql """CREATE TABLES FUNCTION udtf_list(array, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfList", + "type"="JAVA_UDF" + );""" + + sql """DROP FUNCTION IF EXISTS udtf_map(map, int);""" + sql """CREATE TABLES FUNCTION udtf_map(map, int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFAllTypeTest\$UdtfMap", + "type"="JAVA_UDF" + );""" + + qt_select_boolean_col """select int_col, boolean_col, e1 from ${tableName} lateral view udtf_boolean(boolean_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_tinyint_col """select int_col, tinyint_col, e1 from ${tableName} lateral view udtf_tinyint(tinyint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_smallint_col """select int_col, smallint_col, e1 from ${tableName} lateral view udtf_short(smallint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_int_col """select int_col, int_col, e1 from ${tableName} lateral view udtf_int(int_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_bigint_col """select int_col, bigint_col, e1 from ${tableName} lateral view udtf_long(bigint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_largeint_col """select int_col, largeint_col,e1 from ${tableName} lateral view udtf_largeint(largeint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_decimal_col """select int_col, decimal_col,e1 from ${tableName} lateral view udtf_decimal(decimal_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_float_col """select int_col, float_col,e1 from ${tableName} lateral view udtf_float(float_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_double_col """select int_col, double_col,e1 from ${tableName} lateral view udtf_double(double_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_date_col """select int_col, date_col,e1 from ${tableName} lateral view udtf_date(date_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_datetime_col """select int_col, datetime_col,e1 from ${tableName} lateral view udtf_datetime(datetime_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_string_col """select int_col, string_col,e1 from ${tableName} lateral view udtf_string(string_col, "") tmp1 as e1 order by int_col,2,3;""" + qt_select_array_col """select int_col, array_col,e1 from ${tableName} lateral view udtf_list(array_col, int_col) tmp1 as e1 order by int_col,3;""" + qt_select_map_col """select int_col, map_col,e1 from ${tableName} lateral view udtf_map(map_col, int_col) tmp1 as e1 order by int_col,3;""" + qt_select_boolean_col_outer """select int_col, boolean_col,e1 from ${tableName} lateral view udtf_boolean_outer(boolean_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_tinyint_col_outer """select int_col, tinyint_col,e1 from ${tableName} lateral view udtf_tinyint_outer(tinyint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_smallint_col_outer """select int_col, smallint_col,e1 from ${tableName} lateral view udtf_short_outer(smallint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_int_col_outer """select int_col, int_col,e1 from ${tableName} lateral view udtf_int_outer(int_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_bigint_col_outer """select int_col, bigint_col,e1 from ${tableName} lateral view udtf_long_outer(bigint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_largeint_col_outer """select int_col, largeint_col,e1 from ${tableName} lateral view udtf_largeint_outer(largeint_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_decimal_col_outer """select int_col, decimal_col,e1 from ${tableName} lateral view udtf_decimal_outer(decimal_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_float_col_outer """select int_col, float_col,e1 from ${tableName} lateral view udtf_float_outer(float_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_double_col_outer """select int_col, double_col,e1 from ${tableName} lateral view udtf_double_outer(double_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_date_col_outer """select int_col, date_col,e1 from ${tableName} lateral view udtf_date_outer(date_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_datetime_col_outer """select int_col, datetime_col,e1 from ${tableName} lateral view udtf_datetime_outer(datetime_col, int_col) tmp1 as e1 order by int_col,2,3;""" + qt_select_string_col_outer """select int_col, string_col,e1 from ${tableName} lateral view udtf_string_outer(string_col, "") tmp1 as e1 order by int_col,2,3;""" + qt_select_array_col_outer """select int_col, array_col,e1 from ${tableName} lateral view udtf_list_outer(array_col, int_col) tmp1 as e1 order by int_col,3;""" + qt_select_map_col_outer """select int_col, map_col,e1 from ${tableName} lateral view udtf_map_outer(map_col, int_col) tmp1 as e1 order by int_col,3;""" + // qt_java_udf_all_types """select + // int_col, + // udtf_boolean(boolean_col), + // udtf_tinyint(tinyint_col), + // udtf_short(smallint_col), + // udtf_int(int_col), + // udtf_long(bigint_col), + // udtf_largeint(largeint_col), + // udtf_decimal(decimal_col), + // udtf_float(float_col), + // udtf_double(double_col), + // udtf_date(date_col), + // udtf_datetime(datetime_col), + // udtf_string(string_col), + // udtf_list(array_col), + // udtf_map(map_col) + // from ${tableName} order by int_col;""" + } finally { + try_sql """DROP FUNCTION IF EXISTS udtf_boolean_outer(boolean, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_tinyint_outer(tinyint, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_short_outer(smallint, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_int_outer(int, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_long_outer(bigint, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_largeint_outer(largeint, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_decimal_outer(decimal(15, 4), int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_float_outer(float, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_double_outer(double, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_date_outer(date, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_datetime_outer(datetime(6), int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_string_outer(string, string);""" + try_sql """DROP FUNCTION IF EXISTS udtf_list_outer(array, int);""" + try_sql """DROP FUNCTION IF EXISTS udtf_map_outer(map, int);""" + // try_sql("""DROP TABLE IF EXISTS ${tableName};""") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy new file mode 100644 index 00000000000000..c9140c689ebedf --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_arrayint") { + def tableName = "test_javaudtf_arrayint" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `tinyint_col` TINYINT NOT NULL COMMENT "", + `datev2_col` datev2 NOT NULL COMMENT "", + `datetimev2_col` datetimev2 NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i ++) { + sb.append(""" + (${i},${i}*2,'2022-01-01','2022-01-01 11:11:11','a${i}b'), + """) + } + sb.append(""" + (${i},${i}*2,'2022-06-06','2022-01-01 12:12:12','a${i}b') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """DROP FUNCTION IF EXISTS udtf_arrayint_outer(array);""" + sql """ CREATE TABLES FUNCTION udtf_arrayint(array) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFArrayIntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, string_col, e1 FROM ${tableName} lateral view udtf_arrayint(array(user_id)) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_arrayint(array);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy new file mode 100644 index 00000000000000..ae3cacb246a2f7 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_decimal") { + def tableName = "test_javaudtf_decimal" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `cost_1` decimal(27,9) NOT NULL COMMENT "", + `cost_2` decimal(27,9) COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + + + sql """ INSERT INTO ${tableName} (`user_id`,`cost_1`,`cost_2`) VALUES + (111,11111.11111,222222.3333333), + (112,1234556.11111,222222.3333333), + (113,87654321.11111,null) + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + sql """DROP FUNCTION IF EXISTS udtf_decimal_outer(decimal(27,9));""" + sql """ CREATE TABLES FUNCTION udtf_decimal(decimal(27,9)) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFDecimalTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, cost_1, e1 FROM ${tableName} lateral view udtf_decimal(cost_1) temp as e1 order by user_id; """ + qt_select2 """ SELECT user_id, cost_2, e1 FROM ${tableName} lateral view udtf_decimal(cost_2) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_decimal(decimal);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy new file mode 100644 index 00000000000000..22d54f81b8a242 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_double") { + def tableName = "test_javaudtf_double" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `float_1` FLOAT NOT NULL COMMENT "", + `float_2` FLOAT COMMENT "", + `double_1` DOUBLE NOT NULL COMMENT "", + `double_2` DOUBLE COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + + + sql """ INSERT INTO ${tableName} (`user_id`,`float_1`,`float_2`,double_1,double_2) VALUES + (111,11111.11111,222222.3333333,12345678.34455677,1111111.999999999999), + (112,1234556.11111,222222.3333333,222222222.3333333333333,4444444444444.555555555555), + (113,87654321.11111,null,6666666666.6666666666,null) + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """DROP FUNCTION IF EXISTS udtf_double_outer(double);""" + sql """ CREATE TABLES FUNCTION udtf_double(double) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFDoubleTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, double_1, e1 FROM ${tableName} lateral view udtf_double(double_1) temp as e1 order by user_id; """ + qt_select2 """ SELECT user_id, double_2, e1 FROM ${tableName} lateral view udtf_double(double_2) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_double(double);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy new file mode 100644 index 00000000000000..a1d85c99cba375 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_int") { + def tableName = "test_javaudtf_int" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "用户id", + `char_col` CHAR NOT NULL COMMENT "", + `varchar_col` VARCHAR(10) NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + sb.append(""" + (${i % 3}, '${i}','abc,defg','poiuytre,abcdefg'), + """) + } + sb.append(""" + (${i}, '${i}','ab,cdefg','poiuytreabcde,fg') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """DROP FUNCTION IF EXISTS udtf_int_outer(int);""" + sql """ CREATE TABLES FUNCTION udtf_int(int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFIntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_int(user_id) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_int(int);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy new file mode 100644 index 00000000000000..b3f034937fa15d --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_string") { + def tableName = "test_javaudtf_string" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "用户id", + `char_col` CHAR NOT NULL COMMENT "", + `varchar_col` VARCHAR(10) NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + sb.append(""" + (${i % 3}, '${i}','abc,defg','poiuytre,abcdefg'), + """) + } + sb.append(""" + (${i}, '${i}','ab,cdefg','poiuytreabcde,fg') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """DROP FUNCTION IF EXISTS udtf_string_split_outer(string, string);""" + sql """ CREATE TABLES FUNCTION udtf_string_split(string, string) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFStringTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_string_split(varchar_col, ",") temp as e1 order by user_id; """ + + sql """DROP FUNCTION IF EXISTS udtf_null_outer(string, string);""" + sql """ CREATE TABLES FUNCTION udtf_null(string, string) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFNullTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select2 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_null(varchar_col, ",") temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_string_split(string, string);") + try_sql("DROP FUNCTION IF EXISTS udtf_null(string, string);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +}