diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 27efa016fab3d86..492a884f5f9c5e7 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1136,6 +1136,16 @@ DEFINE_mInt32(query_statistics_reserve_timeout_ms, "30000"); DEFINE_mBool(check_segment_when_build_rowset_meta, "false"); +DEFINE_mInt32(max_s3_client_retry, "10"); + +DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors"); + +// ca_cert_file is in this path by default, Normally no modification is required +// ca cert default path is different from different OS +DEFINE_mString(ca_cert_file_paths, + "/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;" + "/etc/ssl/ca-bundle.pem"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index b45c03ed2b97cbc..7af79880fe445d7 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1189,6 +1189,15 @@ DECLARE_mInt32(query_statistics_reserve_timeout_ms); DECLARE_mBool(check_segment_when_build_rowset_meta); +// max s3 client retry times +DECLARE_mInt32(max_s3_client_retry); + +// the directory for storing the trino-connector plugins. +DECLARE_String(trino_connector_plugin_dir); + +// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it to init s3client +DECLARE_mString(ca_cert_file_paths); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/common/status.h b/be/src/common/status.h index 555ee73443d8c34..32f60e72bae1b70 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -83,6 +83,7 @@ E(COPY_FILE_ERROR, -121); E(FILE_ALREADY_EXIST, -122); E(BAD_CAST, -123); E(ARITHMETIC_OVERFLOW_ERRROR, -124); +E(PERMISSION_DENIED, -125); E(CALL_SEQUENCE_ERROR, -202); E(BUFFER_OVERFLOW, -204); E(CONFIG_ERROR, -205); diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp index 648a358850fe848..05817de18524288 100644 --- a/be/src/io/fs/err_utils.cpp +++ b/be/src/io/fs/err_utils.cpp @@ -27,6 +27,8 @@ #include "io/fs/hdfs.h" namespace doris { +using namespace ErrorCode; + namespace io { std::string errno_to_str() { @@ -73,5 +75,39 @@ std::string glob_err_to_str(int code) { return fmt::format("({}), {}", code, msg); } +Status localfs_error(const std::error_code& ec, std::string_view msg) { + if (ec == std::errc::io_error) { + return Status::Error(msg); + } else if (ec == std::errc::no_such_file_or_directory) { + return Status::Error(msg); + } else if (ec == std::errc::file_exists) { + return Status::Error(msg); + } else if (ec == std::errc::no_space_on_device) { + return Status::Error(msg); + } else if (ec == std::errc::permission_denied) { + return Status::Error(msg); + } else { + return Status::Error("{}: {}", msg, ec.message()); + } +} + +Status localfs_error(int posix_errno, std::string_view msg) { + switch (posix_errno) { + case EIO: + return Status::Error(msg); + case ENOENT: + return Status::Error(msg); + case EEXIST: + return Status::Error(msg); + case ENOSPC: + return Status::Error(msg); + case EACCES: + return Status::Error(msg); + default: + return Status::Error("{}: {}", msg, + std::strerror(posix_errno)); + } +} + } // namespace io } // namespace doris diff --git a/be/src/io/fs/err_utils.h b/be/src/io/fs/err_utils.h index 971596fab19f682..42a455fcf2db606 100644 --- a/be/src/io/fs/err_utils.h +++ b/be/src/io/fs/err_utils.h @@ -20,6 +20,8 @@ #include #include +#include "common/status.h" + namespace doris { namespace io { @@ -28,5 +30,8 @@ std::string errcode_to_str(const std::error_code& ec); std::string hdfs_error(); std::string glob_err_to_str(int code); +Status localfs_error(const std::error_code& ec, std::string_view msg); +Status localfs_error(int posix_errno, std::string_view msg); + } // namespace io } // namespace doris diff --git a/be/src/io/fs/local_file_reader.cpp b/be/src/io/fs/local_file_reader.cpp index d0054a1c9703b2a..7a29b39e0fbfb10 100644 --- a/be/src/io/fs/local_file_reader.cpp +++ b/be/src/io/fs/local_file_reader.cpp @@ -68,9 +68,7 @@ Status LocalFileReader::close() { } #endif if (-1 == res) { - std::string err = errno_to_str(); - LOG(WARNING) << fmt::format("failed to close {}: {}", _path.native(), err); - return Status::IOError("failed to close {}: {}", _path.native(), err); + return localfs_error(errno, fmt::format("failed to close {}", _path.native())); } _fd = -1; } @@ -81,8 +79,9 @@ Status LocalFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_ const IOContext* /*io_ctx*/) { DCHECK(!closed()); if (offset > _file_size) { - return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})", - offset, _file_size, _path.native()); + return Status::InternalError( + "offset exceeds file size(offset: {}, file size: {}, path: {})", offset, _file_size, + _path.native()); } size_t bytes_req = result.size; char* to = result.data; @@ -92,10 +91,10 @@ Status LocalFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_ while (bytes_req != 0) { auto res = ::pread(_fd, to, bytes_req, offset); if (UNLIKELY(-1 == res && errno != EINTR)) { - return Status::IOError("cannot read from {}: {}", _path.native(), std::strerror(errno)); + return localfs_error(errno, fmt::format("failed to read {}", _path.native())); } if (UNLIKELY(res == 0)) { - return Status::IOError("cannot read from {}: unexpected EOF", _path.native()); + return Status::InternalError("cannot read from {}: unexpected EOF", _path.native()); } if (res > 0) { to += res; diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index f7ac1e77e162a7e..4e0c9aaefae75d9 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -62,7 +62,7 @@ Status LocalFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer const FileWriterOptions* opts) { int fd = ::open(file.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666); if (-1 == fd) { - return Status::IOError("failed to open {}: {}", file.native(), errno_to_str()); + return localfs_error(errno, fmt::format("failed to create file {}", file.native())); } bool sync_data = opts != nullptr ? opts->sync_file_data : true; *writer = std::make_unique( @@ -81,7 +81,7 @@ Status LocalFileSystem::open_file_impl(const FileDescription& file_desc, const P int fd = -1; RETRY_ON_EINTR(fd, open(abs_path.c_str(), O_RDONLY)); if (fd < 0) { - return Status::IOError("failed to open {}: {}", abs_path.native(), errno_to_str()); + return localfs_error(errno, fmt::format("failed to open {}", abs_path.native())); } *reader = std::make_shared( std::move(abs_path), fsize, fd, @@ -94,13 +94,13 @@ Status LocalFileSystem::create_directory_impl(const Path& dir, bool failed_if_ex bool exists = true; RETURN_IF_ERROR(exists_impl(dir, &exists)); if (exists) { - return Status::IOError("failed to create {}, already exists", dir.native()); + return Status::AlreadyExist("failed to create {}, already exists", dir.native()); } } std::error_code ec; std::filesystem::create_directories(dir, ec); if (ec) { - return Status::IOError("failed to create {}: {}", dir.native(), errcode_to_str(ec)); + return localfs_error(ec, fmt::format("failed to create {}", dir.native())); } return Status::OK(); } @@ -112,12 +112,12 @@ Status LocalFileSystem::delete_file_impl(const Path& file) { return Status::OK(); } if (!std::filesystem::is_regular_file(file)) { - return Status::IOError("failed to delete {}, not a file", file.native()); + return Status::InternalError("failed to delete {}, not a file", file.native()); } std::error_code ec; std::filesystem::remove(file, ec); if (ec) { - return Status::IOError("failed to delete {}: {}", file.native(), errcode_to_str(ec)); + return localfs_error(ec, fmt::format("failed to delete {}", file.native())); } return Status::OK(); } @@ -129,12 +129,12 @@ Status LocalFileSystem::delete_directory_impl(const Path& dir) { return Status::OK(); } if (!std::filesystem::is_directory(dir)) { - return Status::IOError("failed to delete {}, not a directory", dir.native()); + return Status::InternalError("failed to delete {}, not a dir", dir.native()); } std::error_code ec; std::filesystem::remove_all(dir, ec); if (ec) { - return Status::IOError("failed to delete {}: {}", dir.native(), errcode_to_str(ec)); + return localfs_error(ec, fmt::format("failed to delete {}", dir.native())); } return Status::OK(); } @@ -174,7 +174,7 @@ Status LocalFileSystem::file_size_impl(const Path& file, int64_t* file_size) con std::error_code ec; *file_size = std::filesystem::file_size(file, ec); if (ec) { - return Status::IOError("failed to get file size {}: {}", file.native(), errcode_to_str(ec)); + return localfs_error(ec, fmt::format("failed to check exists {}", file.native())); } return Status::OK(); } @@ -208,10 +208,10 @@ Status LocalFileSystem::list_impl(const Path& dir, bool only_file, std::vectorread_at(0, {*content}, &bytes_read)); if (bytes_read != file_size) { - return Status::IOError("failed to read file {} to string. bytes read: {}, file size: {}", - file.native(), bytes_read, file_size); + return Status::InternalError( + "failed to read file {} to string. bytes read: {}, file size: {}", file.native(), + bytes_read, file_size); } return file_reader->close(); } @@ -479,7 +469,7 @@ Status LocalFileSystem::_glob(const std::string& pattern, std::vector 0) { from += res; @@ -171,7 +170,7 @@ Status LocalFileWriter::finalize() { #if defined(__linux__) int flags = SYNC_FILE_RANGE_WRITE; if (sync_file_range(_fd, 0, 0, flags) < 0) { - return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); + return localfs_error(errno, fmt::format("failed to finalize {}", _path.native())); } #endif } @@ -186,14 +185,12 @@ Status LocalFileWriter::_close(bool sync) { if (sync && _dirty) { #ifdef __APPLE__ if (fcntl(_fd, F_FULLFSYNC) < 0) { - return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); - } #else if (0 != ::fdatasync(_fd)) { - return Status::IOError("cannot fdatasync {}: {}", _path.native(), std::strerror(errno)); - } #endif - RETURN_IF_ERROR(detail::sync_dir(_path.parent_path())); + return localfs_error(errno, fmt::format("failed to sync {}", _path.native())); + } + RETURN_IF_ERROR(sync_dir(_path.parent_path())); _dirty = false; } @@ -202,10 +199,9 @@ Status LocalFileWriter::_close(bool sync) { DorisMetrics::instance()->local_bytes_written_total->increment(_bytes_appended); if (0 != ::close(_fd)) { - return Status::IOError("cannot close {}: {}", _path.native(), std::strerror(errno)); + return localfs_error(errno, fmt::format("failed to close {}", _path.native())); } return Status::OK(); } -} // namespace io -} // namespace doris +} // namespace doris::io \ No newline at end of file diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index ffd47b02d7f9561..16069e66a6f3429 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -492,7 +492,11 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader auto st = rowset_reader->next_block(ref_block.get()); if (!st) { if (st.is()) { - eof = true; + if (ref_block->rows() == 0) { + break; + } else { + eof = true; + } } else { return st; } @@ -567,7 +571,11 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea auto st = rowset_reader->next_block(ref_block.get()); if (!st) { if (st.is()) { - eof = true; + if (ref_block->rows() == 0) { + break; + } else { + eof = true; + } } else { return st; } diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index c2bcb302a4fefae..25ba69c04e5de8a 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,18 @@ S3ClientFactory::S3ClientFactory() { return std::make_shared(logLevel); }; Aws::InitAPI(_aws_options); + _ca_cert_file_path = get_valid_ca_cert_path(); +} + +string S3ClientFactory::get_valid_ca_cert_path() { + vector vec_ca_file_path = doris::split(config::ca_cert_file_paths, ";"); + vector::iterator it = vec_ca_file_path.begin(); + for (; it != vec_ca_file_path.end(); ++it) { + if (std::filesystem::exists(*it)) { + return *it; + } + } + return ""; } S3ClientFactory::~S3ClientFactory() { @@ -142,6 +155,16 @@ std::shared_ptr S3ClientFactory::create(const S3Conf& s3_conf Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); aws_config.endpointOverride = s3_conf.endpoint; aws_config.region = s3_conf.region; + std::string ca_cert = get_valid_ca_cert_path(); + if ("" != _ca_cert_file_path) { + aws_config.caFile = _ca_cert_file_path; + } else { + // config::ca_cert_file_paths is valmutable,get newest value if file path invaild + _ca_cert_file_path = get_valid_ca_cert_path(); + if ("" != _ca_cert_file_path) { + aws_config.caFile = _ca_cert_file_path; + } + } if (s3_conf.max_connections > 0) { aws_config.maxConnections = s3_conf.max_connections; } else { diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 393b9d7f1187975..abaf77db6a4a9e5 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -116,10 +116,12 @@ class S3ClientFactory { private: S3ClientFactory(); + static std::string get_valid_ca_cert_path(); Aws::SDKOptions _aws_options; std::mutex _lock; std::unordered_map> _cache; + std::string _ca_cert_file_path; }; } // end namespace doris diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp index 6d0e694f4ab6fdc..23c4f2baeb169fd 100644 --- a/be/src/vec/data_types/data_type_decimal.cpp +++ b/be/src/vec/data_types/data_type_decimal.cpp @@ -162,7 +162,7 @@ bool DataTypeDecimal::parse_from_string(const std::string& str, T* res) const StringParser::ParseResult result = StringParser::PARSE_SUCCESS; *res = StringParser::string_to_decimal::get_primitive_type(), __int128>( str.c_str(), str.size(), precision, scale, &result); - return result == StringParser::PARSE_SUCCESS; + return result == StringParser::PARSE_SUCCESS || result == StringParser::PARSE_UNDERFLOW; } DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_v2) { diff --git a/be/src/vec/exec/vrepeat_node.cpp b/be/src/vec/exec/vrepeat_node.cpp index 58c993c8b3213a6..750f2c3901370a2 100644 --- a/be/src/vec/exec/vrepeat_node.cpp +++ b/be/src/vec/exec/vrepeat_node.cpp @@ -148,8 +148,16 @@ Status VRepeatNode::get_repeated_block(Block* child_block, int repeat_id_idx, Bl } cur_col++; } - + auto rows = _child_block.rows(); // Fill grouping ID to block + RETURN_IF_ERROR(add_grouping_id_column(rows, cur_col, columns, repeat_id_idx)); + DCHECK_EQ(cur_col, column_size); + output_block->set_columns(std::move(columns)); + return Status::OK(); +} + +Status VRepeatNode::add_grouping_id_column(std::size_t rows, std::size_t& cur_col, + vectorized::MutableColumns& columns, int repeat_id_idx) { for (auto slot_idx = 0; slot_idx < _grouping_list.size(); slot_idx++) { DCHECK_LT(slot_idx, _output_tuple_desc->slots().size()); const SlotDescriptor* _virtual_slot_desc = _output_tuple_desc->slots()[cur_col]; @@ -158,16 +166,11 @@ Status VRepeatNode::get_repeated_block(Block* child_block, int repeat_id_idx, Bl int64_t val = _grouping_list[slot_idx][repeat_id_idx]; auto* column_ptr = columns[cur_col].get(); DCHECK(!_output_slots[cur_col]->is_nullable()); - auto* col = assert_cast*>(column_ptr); - for (size_t i = 0; i < child_block->rows(); ++i) { - col->insert_value(val); - } + col->insert_raw_integers(val, rows); cur_col++; } - DCHECK_EQ(cur_col, column_size); - return Status::OK(); } @@ -191,6 +194,17 @@ Status VRepeatNode::pull(doris::RuntimeState* state, vectorized::Block* output_b release_block_memory(_child_block); _repeat_id_idx = 0; } + } else if (_expr_ctxs.empty()) { + auto m_block = vectorized::VectorizedUtils::build_mutable_mem_reuse_block(output_block, + _output_slots); + auto rows = _child_block.rows(); + auto& columns = m_block.mutable_columns(); + + for (int repeat_id_idx = 0; repeat_id_idx < _repeat_id_list.size(); repeat_id_idx++) { + std::size_t cur_col = 0; + RETURN_IF_ERROR(add_grouping_id_column(rows, cur_col, columns, repeat_id_idx)); + } + release_block_memory(_child_block); } RETURN_IF_ERROR(VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); *eos = _child_eos && _child_block.rows() == 0; @@ -202,7 +216,6 @@ Status VRepeatNode::pull(doris::RuntimeState* state, vectorized::Block* output_b Status VRepeatNode::push(RuntimeState* state, vectorized::Block* input_block, bool eos) { _child_eos = eos; DCHECK(!_intermediate_block || _intermediate_block->rows() == 0); - DCHECK(!_expr_ctxs.empty()); if (input_block->rows() > 0) { _intermediate_block = Block::create_unique(); diff --git a/be/src/vec/exec/vrepeat_node.h b/be/src/vec/exec/vrepeat_node.h index 837b4c8aca14e98..32ac9aaa6676838 100644 --- a/be/src/vec/exec/vrepeat_node.h +++ b/be/src/vec/exec/vrepeat_node.h @@ -64,6 +64,9 @@ class VRepeatNode : public ExecNode { private: Status get_repeated_block(Block* child_block, int repeat_id_idx, Block* output_block); + Status add_grouping_id_column(std::size_t rows, std::size_t& cur_col, + vectorized::MutableColumns& columns, int repeat_id_idx); + // Slot id set used to indicate those slots need to set to null. std::vector> _slot_id_set_list; // all slot id diff --git a/be/src/vec/functions/function_jsonb.cpp b/be/src/vec/functions/function_jsonb.cpp index cc0394e2a3ea3e0..f2a24c436489d03 100644 --- a/be/src/vec/functions/function_jsonb.cpp +++ b/be/src/vec/functions/function_jsonb.cpp @@ -1081,7 +1081,8 @@ struct JsonbLengthUtil { MutableColumnPtr res = return_type->create_column(); for (size_t i = 0; i < input_rows_count; ++i) { - if (jsonb_data_column->is_null_at(i) || path_column->is_null_at(i)) { + if (jsonb_data_column->is_null_at(i) || path_column->is_null_at(i) || + (jsonb_data_column->get_data_at(i).size == 0)) { null_map->get_data()[i] = 1; res->insert_data(nullptr, 0); continue; @@ -1101,7 +1102,7 @@ struct JsonbLengthUtil { // doc is NOT necessary to be deleted since JsonbDocument will not allocate memory JsonbDocument* doc = JsonbDocument::createDocument(jsonb_value.data, jsonb_value.size); JsonbValue* value = doc->getValue()->findValue(path, nullptr); - if (UNLIKELY(jsonb_value.size == 0 || !value)) { + if (UNLIKELY(!value)) { null_map->get_data()[i] = 1; res->insert_data(nullptr, 0); continue; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/.test_hidden_dir/wrong_file b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/.test_hidden_dir/wrong_file new file mode 100644 index 000000000000000..d37af4bbc510b4e --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/.test_hidden_dir/wrong_file @@ -0,0 +1 @@ +wrong file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/test_hidden_file/.hidden_file b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/test_hidden_file/.hidden_file new file mode 100644 index 000000000000000..136c05e0d0290e2 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_all_types/test_hidden_file/.hidden_file @@ -0,0 +1 @@ +hidden diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java index 3a08fce57559940..c24b1587065bf6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java @@ -54,14 +54,14 @@ public class FunctionRegistry { // to record the global alias function and other udf. private static final String GLOBAL_FUNCTION = "__GLOBAL_FUNCTION__"; - private final Map> name2InternalBuiltinBuilders; + private final Map> name2BuiltinBuilders; private final Map>> name2UdfBuilders; public FunctionRegistry() { - name2InternalBuiltinBuilders = new ConcurrentHashMap<>(); + name2BuiltinBuilders = new ConcurrentHashMap<>(); name2UdfBuilders = new ConcurrentHashMap<>(); - registerBuiltinFunctions(name2InternalBuiltinBuilders); - afterRegisterBuiltinFunctions(name2InternalBuiltinBuilders); + registerBuiltinFunctions(name2BuiltinBuilders); + afterRegisterBuiltinFunctions(name2BuiltinBuilders); } // this function is used to test. @@ -79,12 +79,33 @@ public FunctionBuilder findFunctionBuilder(String name, Object argument) { } public Optional> tryGetBuiltinBuilders(String name) { - List builders = name2InternalBuiltinBuilders.get(name); - return name2InternalBuiltinBuilders.get(name) == null + List builders = name2BuiltinBuilders.get(name); + return name2BuiltinBuilders.get(name) == null ? Optional.empty() : Optional.of(ImmutableList.copyOf(builders)); } + public boolean isAggregateFunction(String dbName, String name) { + name = name.toLowerCase(); + Class aggClass = org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction.class; + if (StringUtils.isEmpty(dbName)) { + List functionBuilders = name2BuiltinBuilders.get(name); + for (FunctionBuilder functionBuilder : functionBuilders) { + if (aggClass.isAssignableFrom(functionBuilder.functionClass())) { + return true; + } + } + } + + List udfBuilders = findUdfBuilder(dbName, name); + for (FunctionBuilder udfBuilder : udfBuilders) { + if (aggClass.isAssignableFrom(udfBuilder.functionClass())) { + return true; + } + } + return false; + } + // currently we only find function by name and arity and args' types. public FunctionBuilder findFunctionBuilder(String dbName, String name, List arguments) { List functionBuilders = null; @@ -93,11 +114,11 @@ public FunctionBuilder findFunctionBuilder(String dbName, String name, List a if (StringUtils.isEmpty(dbName)) { // search internal function only if dbName is empty - functionBuilders = name2InternalBuiltinBuilders.get(name.toLowerCase()); + functionBuilders = name2BuiltinBuilders.get(name.toLowerCase()); if (CollectionUtils.isEmpty(functionBuilders) && AggStateFunctionBuilder.isAggStateCombinator(name)) { String nestedName = AggStateFunctionBuilder.getNestedName(name); String combinatorSuffix = AggStateFunctionBuilder.getCombinatorSuffix(name); - functionBuilders = name2InternalBuiltinBuilders.get(nestedName.toLowerCase()); + functionBuilders = name2BuiltinBuilders.get(nestedName.toLowerCase()); if (functionBuilders != null) { functionBuilders = functionBuilders.stream() .map(builder -> new AggStateFunctionBuilder(combinatorSuffix, builder)) @@ -193,8 +214,8 @@ public void dropUdf(String dbName, String name, List argTypes) { } synchronized (name2UdfBuilders) { Map> builders = name2UdfBuilders.getOrDefault(dbName, ImmutableMap.of()); - builders.getOrDefault(name, Lists.newArrayList()).removeIf(builder -> ((UdfBuilder) builder).getArgTypes() - .equals(argTypes)); + builders.getOrDefault(name, Lists.newArrayList()) + .removeIf(builder -> ((UdfBuilder) builder).getArgTypes().equals(argTypes)); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 3e98b887eeb8251..f639109356987fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -53,6 +53,7 @@ import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId; import org.apache.doris.planner.external.FileSplit; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; @@ -66,7 +67,6 @@ import com.google.common.collect.Streams; import com.google.common.collect.TreeRangeMap; import lombok.Data; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -1060,14 +1060,16 @@ public void setAcidInfo(AcidInfo acidInfo) { this.acidInfo = acidInfo; } - private boolean isFileVisible(Path path) { - if (path == null || StringUtils.isEmpty(path.toString())) { + @VisibleForTesting + public static boolean isFileVisible(Path path) { + if (path == null) { return false; } - if (path.getName().startsWith(".") || path.getName().startsWith("_")) { + String pathStr = path.toUri().toString(); + if (containsHiddenPath(pathStr) || path.getName().startsWith("_")) { return false; } - for (String name : path.toString().split("/")) { + for (String name : pathStr.split("/")) { if (isGeneratedPath(name)) { return false; } @@ -1075,6 +1077,18 @@ private boolean isFileVisible(Path path) { return true; } + private static boolean containsHiddenPath(String path) { + if (path.startsWith(".")) { + return true; + } + for (int i = 0; i < path.length() - 1; i++) { + if (path.charAt(i) == '/' && path.charAt(i + 1) == '.') { + return true; + } + } + return false; + } + private static boolean isGeneratedPath(String name) { return "_temporary".equals(name) // generated by spark || "_imapala_insert_staging".equals(name) // generated by impala diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java index 4d1abbbf951ba15..39c1eab1383f1e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java @@ -494,6 +494,11 @@ public Plan generateLeadingJoinPlan() { stackTopLevel = currentLevel; } } + if (stack.size() > 1) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("please check your brace pairs in leading"); + return null; + } LogicalJoin finalJoin = (LogicalJoin) stack.pop().second; // we want all filters been remove diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 67a82edbf53eef1..aa44674cf9803c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -485,10 +485,32 @@ protected boolean condition(Rule rule, Plan plan) { logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> { LogicalHaving> having = ctx.root; Aggregate childPlan = having.child(); + + FunctionRegistry functionRegistry + = ctx.cascadesContext.getConnectContext().getEnv().getFunctionRegistry(); + + List groupByExprs = childPlan.getGroupByExpressions(); + Builder groupBySlotsBuilder = ImmutableList.builderWithExpectedSize(groupByExprs.size()); + for (Expression groupBy : groupByExprs) { + if (groupBy instanceof Slot) { + groupBySlotsBuilder.add((Slot) groupBy); + } + } + List groupBySlots = groupBySlotsBuilder.build(); + Set boundConjuncts = having.getConjuncts().stream() .map(expr -> { - expr = bindSlot(expr, childPlan.child(), ctx.cascadesContext, false); - return bindSlot(expr, childPlan, ctx.cascadesContext, false); + if (hasAggregateFunction(expr, functionRegistry)) { + expr = bindSlot(expr, childPlan.child(), ctx.cascadesContext, false); + } else { + expr = new SlotBinder(toScope(ctx.cascadesContext, groupBySlots), + ctx.cascadesContext, false, false + ).bind(expr); + + expr = bindSlot(expr, childPlan, ctx.cascadesContext, false); + expr = bindSlot(expr, childPlan.children(), ctx.cascadesContext, false); + } + return expr; }) .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) @@ -550,7 +572,7 @@ protected boolean condition(Rule rule, Plan plan) { // we need to do cast before set operation, because we maybe use these slot to do shuffle // so, we must cast it before shuffle to get correct hash code. List> childrenProjections = setOperation.collectChildrenProjections(); - ImmutableList.Builder> childrenOutputs = ImmutableList.builder(); + Builder> childrenOutputs = ImmutableList.builder(); Builder newChildren = ImmutableList.builder(); for (int i = 0; i < childrenProjections.size(); i++) { Plan newChild; @@ -824,4 +846,23 @@ private E checkBound(E expression, Plan plan) { }); return expression; } + + private boolean hasAggregateFunction(Expression expression, FunctionRegistry functionRegistry) { + return expression.anyMatch(expr -> { + if (expr instanceof AggregateFunction) { + return true; + } else if (expr instanceof UnboundFunction) { + UnboundFunction unboundFunction = (UnboundFunction) expr; + boolean isAggregateFunction = functionRegistry + .isAggregateFunction( + unboundFunction.getDbName(), + unboundFunction.getName() + ); + if (isAggregateFunction) { + return true; + } + } + return false; + }); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java index 48d3ef0bfea7004..eaab0e3e11f882d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java @@ -58,6 +58,9 @@ public List buildRules() { leading.setStatus(Hint.HintStatus.UNUSED); leading.setErrorMessage("condition does not matched joinType"); } + Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); + Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); + join.setBitmap(LongBitmap.or(leftHand, rightHand)); List expressions = join.getHashJoinConjuncts(); Long totalFilterBitMap = 0L; Long nonNullableSlotBitMap = 0L; @@ -66,6 +69,9 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); + if (join.getJoinType().isLeftJoin()) { + filterBitMap = LongBitmap.or(filterBitMap, rightHand); + } leading.getFilters().add(Pair.of(filterBitMap, expression)); leading.putConditionJoinType(expression, join.getJoinType()); } @@ -75,12 +81,12 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); + if (join.getJoinType().isLeftJoin()) { + filterBitMap = LongBitmap.or(filterBitMap, rightHand); + } leading.getFilters().add(Pair.of(filterBitMap, expression)); leading.putConditionJoinType(expression, join.getJoinType()); } - Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); - Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); - join.setBitmap(LongBitmap.or(leftHand, rightHand)); collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap); return ctx.root; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java index 73bb42ccf0cd6c7..06d85e660b090ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java @@ -121,9 +121,19 @@ private void extractLeading(SelectHintLeading selectHint, CascadesContext contex context.setLeadingJoin(false); return; } - hint.setStatus(Hint.HintStatus.SUCCESS); statementContext.addHint(hint); context.getHintMap().put("Leading", hint); + if (hint.getTablelist().size() < 2) { + hint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + context.getHintMap().get("Leading").setStatus(Hint.HintStatus.UNUSED); + hint.setErrorMessage("less than two tables is not allowed in leading clause"); + statementContext.addHint(hint); + context.setLeadingJoin(false); + return; + } + if (!hint.isSyntaxError()) { + hint.setStatus(Hint.HintStatus.SUCCESS); + } if (hints.get("ordered") != null || ConnectContext.get().getSessionVariable().isDisableJoinReorder()) { context.setLeadingJoin(false); hint.setStatus(Hint.HintStatus.UNUSED); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java index 054aa4767b60183..7b84e646785ab7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java @@ -50,6 +50,11 @@ public AggStateFunctionBuilder(String combinatorSuffix, FunctionBuilder nestedBu this.nestedBuilder = Objects.requireNonNull(nestedBuilder, "nestedBuilder can not be null"); } + @Override + public Class functionClass() { + return nestedBuilder.functionClass(); + } + @Override public boolean canApply(List arguments) { if (combinatorSuffix.equals(STATE)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BuiltinFunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BuiltinFunctionBuilder.java index 74c4a918cf0aa0a..45cabb8d55ed14e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BuiltinFunctionBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BuiltinFunctionBuilder.java @@ -42,13 +42,21 @@ public class BuiltinFunctionBuilder extends FunctionBuilder { // Concrete BoundFunction's constructor private final Constructor builderMethod; + private final Class functionClass; - public BuiltinFunctionBuilder(Constructor builderMethod) { + public BuiltinFunctionBuilder( + Class functionClass, Constructor builderMethod) { + this.functionClass = Objects.requireNonNull(functionClass, "functionClass can not be null"); this.builderMethod = Objects.requireNonNull(builderMethod, "builderMethod can not be null"); this.arity = builderMethod.getParameterCount(); this.isVariableLength = arity > 0 && builderMethod.getParameterTypes()[arity - 1].isArray(); } + @Override + public Class functionClass() { + return functionClass; + } + @Override public boolean canApply(List arguments) { if (isVariableLength && arity > arguments.size() + 1) { @@ -133,7 +141,9 @@ public static List resolve(Class funct + functionClass.getSimpleName()); return Arrays.stream(functionClass.getConstructors()) .filter(constructor -> Modifier.isPublic(constructor.getModifiers())) - .map(constructor -> new BuiltinFunctionBuilder((Constructor) constructor)) + .map(constructor -> new BuiltinFunctionBuilder( + functionClass, (Constructor) constructor) + ) .collect(ImmutableList.toImmutableList()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/FunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/FunctionBuilder.java index 0b3a50a239b8e0b..d1e69d3e307d6f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/FunctionBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/FunctionBuilder.java @@ -27,6 +27,8 @@ * This class used to build BoundFunction(Builtin or Combinator) by a list of Expressions. */ public abstract class FunctionBuilder { + public abstract Class functionClass(); + /** check whether arguments can apply to the constructor */ public abstract boolean canApply(List arguments); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java index 197cb8b396df5d3..733bd5fcae1164a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java @@ -50,6 +50,11 @@ public List getArgTypes() { return aliasUdf.getArgTypes(); } + @Override + public Class functionClass() { + return AliasUdf.class; + } + @Override public boolean canApply(List arguments) { if (arguments.size() != aliasUdf.arity()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java index d0c0b067e944d21..89073b4eb77e4dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java @@ -49,6 +49,11 @@ public List getArgTypes() { .collect(Collectors.toList())).get(); } + @Override + public Class functionClass() { + return JavaUdaf.class; + } + @Override public boolean canApply(List arguments) { if ((isVarArgs && arity > arguments.size() + 1) || (!isVarArgs && arguments.size() != arity)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java index c11b07665a6ebf8..5df0690c6609355 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java @@ -52,6 +52,11 @@ public List getArgTypes() { .collect(Collectors.toList())).get(); } + @Override + public Class functionClass() { + return JavaUdf.class; + } + @Override public boolean canApply(List arguments) { if ((isVarArgs && arity > arguments.size() + 1) || (!isVarArgs && arguments.size() != arity)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java new file mode 100644 index 000000000000000..0937bbc3cc856cb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.datasource.hive.HiveMetaStoreCache.FileCacheValue; + +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Test; + +public class PathVisibleTest { + @Test + public void shouldReturnFalseWhenPathIsNull() { + Assert.assertFalse(FileCacheValue.isFileVisible(null)); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/.hidden/path"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible/path/.file"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_temporary_xx"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_imapala_insert_staging"))); + + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible//.hidden/path"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("///visible/path/.file"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path///_temporary_xx"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_imapala_insert_staging"))); + + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("s3://visible/path"))); + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path"))); + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://visible/path./1.txt"))); + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt"))); + } +} diff --git a/regression-test/data/correctness_p0/test_grouping_sets_empty.out b/regression-test/data/correctness_p0/test_grouping_sets_empty.out new file mode 100644 index 000000000000000..fde7e6420dab8bf --- /dev/null +++ b/regression-test/data/correctness_p0/test_grouping_sets_empty.out @@ -0,0 +1,39 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +3 + +-- !select2 -- +3 + +-- !select3 -- +3 +3 +3 + +-- !select4 -- +\N +\N +\N +1 +2 +3 + +-- !select5 -- +3 + +-- !select6 -- +3 + +-- !select7 -- +3 +3 +3 + +-- !select8 -- +\N +\N +\N +1 +2 +3 + diff --git a/regression-test/data/datatype_p0/decimalv3/test_load.out b/regression-test/data/datatype_p0/decimalv3/test_load.out index 6907d61a12af837..663c786ea8e957f 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_load.out +++ b/regression-test/data/datatype_p0/decimalv3/test_load.out @@ -8,3 +8,31 @@ -99999999999999999999999999999999.999999 4.000000 99999999999999999999999999999999.999999 3.000000 +-- !decimalv3_underflow1 -- +1.00100 2.00200 +1.00200 0.00000 +1.00300 0.10000 +1.00400 0.10004 +1.00500 0.10005 + +-- !decimalv3_underflow2 -- +1.00100 0.00000 +1.00200 0.00000 +1.00300 0.10000 +1.00400 0.10004 +1.00500 0.10005 + +-- !decimalv3_underflow3 -- +1.00100 2.00200 +1.00200 0.00000 +1.00300 0.10000 +1.00400 0.10004 +1.00500 0.10005 + +-- !decimalv3_underflow4 -- +1.00100 0.00000 +1.00200 0.00000 +1.00300 0.10000 +1.00400 0.10004 +1.00500 0.10005 + diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index 215f798df9b68ff..8a3b573cebdba2b 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -7494,6 +7494,9 @@ false -- !sql_json_length -- 2 +-- !sql_json_length -- +\N + -- !select_length -- 1 \N \N 2 null 1 diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out index 4eb83a9abd478a4..95e8befe6d9c802 100644 --- a/regression-test/data/nereids_p0/hint/fix_leading.out +++ b/regression-test/data/nereids_p0/hint/fix_leading.out @@ -209,3 +209,20 @@ SyntaxError: -- !select2_5_13 -- 816000 +-- !select3_1 -- +1719 + +-- !select3_2 -- +1719 + +-- !select3_3 -- +1719 + +-- !select3_4 -- +5169 + +-- !select4_1 -- +1000 + +-- !select4_2 -- +1000 diff --git a/regression-test/data/nereids_syntax_p0/bind_priority.out b/regression-test/data/nereids_syntax_p0/bind_priority.out index 7cd1e3ea5696e43..b3bc666c23d50ed 100644 --- a/regression-test/data/nereids_syntax_p0/bind_priority.out +++ b/regression-test/data/nereids_syntax_p0/bind_priority.out @@ -36,3 +36,28 @@ all 2 4 5 6 6 +-- !having_bind_child -- +1 10 + +-- !having_bind_child2 -- +2 10 + +-- !having_bind_child3 -- +2 10 + +-- !having_bind_project -- +2 10 + +-- !having_bind_project2 -- + +-- !having_bind_project3 -- + +-- !having_bind_project4 -- +2 11 + +-- !having_bind_child4 -- +2 11 + +-- !having_bind_child5 -- +2 11 + diff --git a/regression-test/suites/correctness_p0/test_grouping_sets_empty.groovy b/regression-test/suites/correctness_p0/test_grouping_sets_empty.groovy new file mode 100644 index 000000000000000..b0bd6b19039203f --- /dev/null +++ b/regression-test/suites/correctness_p0/test_grouping_sets_empty.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_grouping_sets_empty") { + + sql""" + drop table if exists test_grouping_sets_empty; + """ + + sql""" + create table test_grouping_sets_empty (a int) distributed by hash(a) buckets 1 properties ( 'replication_num' = '1'); + """ + + sql """ + insert into test_grouping_sets_empty values (1),(2),(3); + """ + + qt_select1 """ + select count(a) from test_grouping_sets_empty group by grouping sets (()); + """ + + + qt_select2 """ + select count(*) from test_grouping_sets_empty group by grouping sets (()); + """ + + qt_select3 """ + select count(*) from test_grouping_sets_empty group by grouping sets ((),(),()); + """ + + qt_select4 """ + select a from test_grouping_sets_empty group by grouping sets ((),(),(),(a)) order by a; + """ + + + qt_select5 """ + select count(a) from test_grouping_sets_empty group by grouping sets (()); + """ + + + qt_select6 """ + select count(*) from test_grouping_sets_empty group by grouping sets (()); + """ + + qt_select7 """ + select count(*) from test_grouping_sets_empty group by grouping sets ((),(),()); + """ + + qt_select8 """ + select a from test_grouping_sets_empty group by grouping sets ((),(),(),(a)) order by a; + """ + + + +} diff --git a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy index 74de23bcf9cc056..1c0c95e53257683 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy @@ -104,4 +104,59 @@ suite("test_load") { exception "error" } qt_decimalv3_insert "select * from test_decimalv3_insert order by 1, 2;" + + // fix + sql """ set enable_nereids_dml=true; """ + sql """ + drop TABLE if exists test_sys_update_basic_test_update_decimal_tb; + """ + sql """ + CREATE TABLE test_sys_update_basic_test_update_decimal_tb ( + k1 DECIMAL(10, 5) NULL, + v1 DECIMAL(10, 5) NULL + ) UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 5 PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ + insert into test_sys_update_basic_test_update_decimal_tb values + (1.001, 2.002), (1.002, 0.00000002), (1.003, 0.100000001), (1.004, 0.100044001), (1.005, 0.100045001); + """ + qt_decimalv3_underflow1 """ + select * from test_sys_update_basic_test_update_decimal_tb order by 1, 2; + """ + sql """ + UPDATE test_sys_update_basic_test_update_decimal_tb SET v1="0.00000001" WHERE k1 = 1.001; + """ + qt_decimalv3_underflow2 """ + select * from test_sys_update_basic_test_update_decimal_tb order by 1, 2; + """ + + sql """ set enable_nereids_dml=false; """ + sql """ + drop TABLE if exists test_sys_update_basic_test_update_decimal_tb; + """ + sql """ + CREATE TABLE test_sys_update_basic_test_update_decimal_tb ( + k1 DECIMAL(10, 5) NULL, + v1 DECIMAL(10, 5) NULL + ) UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 5 PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ + insert into test_sys_update_basic_test_update_decimal_tb values + (1.001, 2.002), (1.002, 0.00000002), (1.003, 0.100000001), (1.004, 0.100044001), (1.005, 0.100045001); + """ + qt_decimalv3_underflow3 """ + select * from test_sys_update_basic_test_update_decimal_tb order by 1, 2; + """ + // need to use "1.001"(quoted) because non-nereids dml seems treat 1.001 as double, + // which will cause wrong result of decimal comparision + sql """ + UPDATE test_sys_update_basic_test_update_decimal_tb SET v1="0.00000001" WHERE k1 = "1.001"; + """ + qt_decimalv3_underflow4 """ + select * from test_sys_update_basic_test_update_decimal_tb order by 1, 2; + """ } diff --git a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy index 1d775a46ffa22c0..491d642be4e81b9 100644 --- a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy +++ b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy @@ -548,6 +548,7 @@ suite("test_jsonb_load_and_function", "p0") { qt_sql_json_length """SELECT json_length('{"k1":"v31","k2":300}')""" qt_sql_json_length """SELECT json_length('{"a.b.c":{"k1.a1":"v31", "k2": 300},"a":"niu"}')""" qt_sql_json_length """SELECT json_length('{"a":{"k1.a1":"v31", "k2": 300},"b":"niu"}','\$.a')""" + qt_sql_json_length """SELECT json_length('abc','\$.k1')""" qt_select_length """SELECT id, j, json_length(j) FROM ${testTable} ORDER BY id""" qt_select_length """SELECT id, j, json_length(j, '\$[1]') FROM ${testTable} ORDER BY id""" diff --git a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy new file mode 100644 index 000000000000000..776bf2ecf0bd17f --- /dev/null +++ b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +suite("test_create_mv") { + def tableName = "test_mv_10010" + + def getJobState = { table -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${table}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `load_time` datetime NOT NULL COMMENT '事件发生时间', + `id` varchar(192) NOT NULL COMMENT '', + `class` varchar(192) NOT NULL COMMENT '', + `result` int NOT NULL COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`load_time`,`id`, `class`) + COMMENT '' + PARTITION BY RANGE(`load_time`)( + PARTITION p1 VALUES LESS THAN ("2025-01-01 00:00:00") + ) + DISTRIBUTED BY HASH(`load_time`,`id`, `class`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ insert into ${tableName} values ('2024-03-20 10:00:00', 'a', 'b', 1) """ + + sql """ + create materialized view mv_1 as + select + date_trunc(load_time, 'minute'), + id, + class, + count(id) as total, + min(result) as min_result, + sum(result) as max_result + from + ${tableName} + group by date_trunc(load_time, 'minute'), id, class; + """ + + sql """ SHOW ALTER TABLE MATERIALIZED VIEW """ + + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tableName) + if (res == "FINISHED" || res == "CANCELLED") { + assertEquals("FINISHED", res) + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } +} diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 782bfc346479a49..583a21d8700ef1d 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -150,4 +150,20 @@ suite("fix_leading") { qt_select2_5_11 """select /*+ leading(t3 {t1 t2}) */ count(*) from t1 right join t2 on c2 = c2 left semi join t3 on c2 = c3;""" qt_select2_5_12 """select /*+ leading(t3 t2 t1) */ count(*) from t1 right join t2 on c2 = c2 left semi join t3 on c2 = c3;""" qt_select2_5_13 """select /*+ leading(t3 {t2 t1}) */ count(*) from t1 right join t2 on c2 = c2 left semi join t3 on c2 = c3;""" + + // check only one table used in leading + qt_select3_1 """select /*+ leading(t1) */ count(*) from t1 join t2 on c1 = c2;""" + + // check only one table used in leading and add brace + qt_select3_2 """select /*+ leading({t1}) */ count(*) from t1 join t2 on c1 = c2;""" + + // check mistake usage of brace + qt_select3_3 """select /*+ leading(t1 {t2}) */ count(*) from t1 join t2 on c1 = c2;""" + + // check using subquery alias to cte in cte query + qt_select3_4 """with cte as (select c1 from t1) select count(*) from t1 join (select /*+ leading(cte t2) */ c2 from t2 join cte on c2 = cte.c1) as alias on t1.c1 = alias.c2;""" + + // check left right join result + qt_select4_1 """select count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" } diff --git a/regression-test/suites/nereids_syntax_p0/bind_priority.groovy b/regression-test/suites/nereids_syntax_p0/bind_priority.groovy index 19b630854cb855c..0e02f01a18b1bcb 100644 --- a/regression-test/suites/nereids_syntax_p0/bind_priority.groovy +++ b/regression-test/suites/nereids_syntax_p0/bind_priority.groovy @@ -31,7 +31,7 @@ suite("bind_priority") { sql """ insert into bind_priority_tbl values(1, 2),(3, 4) """ - + sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" @@ -100,17 +100,17 @@ suite("bind_priority") { ); """ sql "insert into bind_priority_tbl2 values(3,5),(2, 6),(1,4);" - + qt_bind_order_to_project_alias """ select bind_priority_tbl.b b, bind_priority_tbl2.b - from bind_priority_tbl join bind_priority_tbl2 on bind_priority_tbl.a=bind_priority_tbl2.a + from bind_priority_tbl join bind_priority_tbl2 on bind_priority_tbl.a=bind_priority_tbl2.a order by b; """ qt_bind_order_to_project_alias """ select bind_priority_tbl.b, bind_priority_tbl2.b b - from bind_priority_tbl join bind_priority_tbl2 on bind_priority_tbl.a=bind_priority_tbl2.a + from bind_priority_tbl join bind_priority_tbl2 on bind_priority_tbl.a=bind_priority_tbl2.a order by b; """ @@ -148,11 +148,86 @@ suite("bind_priority") { ) a ), tb2 as ( - select * from tb1 + select * from tb1 ) select * from tb2 order by id; """) result([[1], [2], [3]]) } + + def testBindHaving = { + sql "drop table if exists test_bind_having_slots" + + sql "create table test_bind_having_slots " + + "(id int, age int) " + + "distributed by hash(id) " + + "properties('replication_num'='1');" + sql "insert into test_bind_having_slots values(1, 10), (2, 20), (3, 30);" + + order_qt_having_bind_child """ + select id, sum(age) + from test_bind_having_slots s + group by id + having id = 1; -- bind id from group by + """ + + order_qt_having_bind_child2 """ + select id + 1 as id, sum(age) + from test_bind_having_slots s + group by id + having id = 1; -- bind id from group by + """ + + + order_qt_having_bind_child3 """ + select id + 1 as id, sum(age) + from test_bind_having_slots s + group by id + having id + 1 = 2; -- bind id from group by + """ + + order_qt_having_bind_project """ + select id + 1 as id, sum(age) + from test_bind_having_slots s + group by id + 1 + having id = 2; -- bind id from project + """ + + order_qt_having_bind_project2 """ + select id + 1 as id, sum(age) + from test_bind_having_slots s + group by id + 1 + having id + 1 = 2; -- bind id from project + """ + + + order_qt_having_bind_project3 """ + select id + 1 as id, sum(age + 1) as age + from test_bind_having_slots s + group by id + having age = 10; -- bind id from age + """ + + order_qt_having_bind_project4 """ + select id + 1 as id, sum(age + 1) as age + from test_bind_having_slots s + group by id + having age = 11; -- bind age from project + """ + + order_qt_having_bind_child4 """ + select id + 1 as id, sum(age + 1) as age + from test_bind_having_slots s + group by id + having sum(age) = 10; -- bind age from s + """ + + order_qt_having_bind_child5 """ + select id + 1 as id, sum(age + 1) as age + from test_bind_having_slots s + group by id + having sum(age + 1) = 11 -- bind age from s + """ + }() }